From 4ce0ba61d5a5cdb49e639db01b1c0b9939aa46b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 19:06:05 +0000 Subject: [PATCH 1/5] feat(memory): add the allocator seam and the external arData install API Every persistent structure z-engine mints allocated through a hardcoded Core::trackedNew(..., persistent) - FFI's pemalloc, i.e. the process heap - which a consumer that keeps its objects in a fork-shared mmap arena cannot redirect. The new ZEngine\Memory\Allocator interface is that seam: it hands out ADDRESSES of zeroed, aligned blocks (no FFI\CData crosses the boundary, so an arena implementation in a consumer package binds its own mmap primitives and returns plain integers) and reports whether it keeps ownership of what it hands out. EngineAllocator is the default implementation and reproduces the three allocation shapes the framework used to hardcode - tracked persistent, tracked request-lifetime and untracked persistent - so a caller that passes nothing gets byte-for-byte the behavior it had before. HashTable's constructor takes the optional allocator and refuses destroy() when the memory belongs to a foreign allocator: both frees assume z-engine's own allocator and would corrupt an arena. PersistentHashTable can now be handed the OTHER half of a table's memory: installExternalStorage() writes the engine's own zend_hash_real_init_mixed layout (hash part of two uint32_t slots per bucket reset to HT_INVALID_IDX, buckets right behind it) over a caller-allocated block, before the first insert makes the engine allocate storage of its own. Because the engine grows a full table by perealloc()ing that very block, growth is guarded from both sides: inserts through the wrapper refuse the write that would trigger the resize, and assertNoGrowth() diagnoses a relocation that engine paths caused behind the wrapper's back. Refs #223 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B9xaBchjdo1atarNZ6sZqe --- src/Memory/AllocationException.php | 51 ++++++ src/Memory/Allocator.php | 91 ++++++++++ src/Memory/EngineAllocator.php | 126 +++++++++++++ src/Type/HashTable.php | 57 +++++- src/Type/PersistentHashTable.php | 270 ++++++++++++++++++++++++++++ src/Type/TypeOperationException.php | 73 ++++++++ 6 files changed, 665 insertions(+), 3 deletions(-) create mode 100644 src/Memory/AllocationException.php create mode 100644 src/Memory/Allocator.php create mode 100644 src/Memory/EngineAllocator.php diff --git a/src/Memory/AllocationException.php b/src/Memory/AllocationException.php new file mode 100644 index 0000000..f84ecb1 --- /dev/null +++ b/src/Memory/AllocationException.php @@ -0,0 +1,51 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Memory; + +/** + * Raised when an Allocator cannot satisfy a request + * + * These failures happen BEFORE a single byte is handed out, so a rejected allocation + * leaves neither the allocator nor the structure that asked for memory in a half-built + * state. Every failure mode has a named static constructor (project convention, see + * AGENTS.md). + */ +class AllocationException extends \RuntimeException +{ + /** + * Raised when the requested block size is not a positive number of bytes + */ + public static function invalidSize(int $size): self + { + return new self("An allocation size must be a positive number of bytes, {$size} given"); + } + + /** + * Raised when the requested alignment is not a power of two + */ + public static function invalidAlignment(int $alignment): self + { + return new self("An allocation alignment must be a power of two, {$alignment} given"); + } + + /** + * Raised when the allocator cannot guarantee the alignment the caller asked for + */ + public static function unsupportedAlignment(int $requested, int $guaranteed): self + { + return new self( + "This allocator guarantees {$guaranteed}-byte alignment, {$requested} bytes were requested", + ); + } +} diff --git a/src/Memory/Allocator.php b/src/Memory/Allocator.php new file mode 100644 index 0000000..b1e88a0 --- /dev/null +++ b/src/Memory/Allocator.php @@ -0,0 +1,91 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Memory; + +/** + * Source of raw, zeroed memory blocks for the persistent structures z-engine mints + * + * Every persistent primitive of the framework - the object clones of + * PersistentObjectFactory, the interned blocks of StringEntry::persistent(), the struct + * of a PersistentHashTable - allocates through one of these. The default implementation + * (EngineAllocator) is the malloc-backed FFI allocator z-engine has always used, so a + * caller that passes nothing keeps exactly the behavior it had before this seam existed. + * + * The seam exists for sinks that are NOT the process heap: a fork-shared mmap arena, a + * shared-memory segment, a preallocated slab. Such an allocator hands out addresses inside + * memory it owns itself, which is why ownsAllocations() is part of the contract: z-engine + * must never release a block it did not allocate (destroy() refuses instead of calling + * free(3) on somebody else's arena). + * + * The interface deliberately speaks in ADDRESSES rather than FFI\CData: a public API of + * this framework never leaks engine handles (see AGENTS.md), and an implementation living + * in a consumer package can therefore be written against nothing but integers - it binds + * its own mmap/shm primitives with FFI::cdef and returns the address it computed. + * + * Implementation contract: + * + * - the returned block is at least $size bytes long and ZEROED, exactly like + * FFI::new()/pemalloc + memset - engine structures are initialized field by field and + * every field the caller does not write must read as zero; + * - the address is aligned to at least $alignment bytes; + * - the block stays alive at least until the owner releases it: nothing in z-engine + * reclaims blocks of a foreign allocator; + * - allocate() throws (AllocationException or an implementation-specific exception) when + * it cannot satisfy the request; it never returns 0. + */ +interface Allocator +{ + /** + * Alignment good enough for every engine structure: what malloc() guarantees on the + * supported platforms (alignof(max_align_t) is 16 on x64 and arm64) + */ + public const int DEFAULT_ALIGNMENT = 16; + + /** + * Alignment an engine structure actually REQUIRES: pointer alignment + * + * zend_object, zend_string, HashTable and Bucket are made of pointers, zend_longs and + * doubles - all of them 8-byte-aligned members on every supported platform, which is + * also what the Zend memory manager itself guarantees (ZEND_MM_ALIGNMENT). The + * framework's own allocations ask for exactly this, so a request-lifetime block is a + * legal answer too; DEFAULT_ALIGNMENT stays the safer public default for callers who + * do not want to reason about it. + */ + public const int ENGINE_STRUCT_ALIGNMENT = 8; + + /** + * Allocates one zeroed block and returns its ADDRESS + * + * @param int $size Number of bytes to allocate, greater than zero + * @param int $alignment Required alignment of the returned address, a power of two + * + * @return int Address of the block, never zero + * + * @throws AllocationException when the request cannot be satisfied + */ + public function allocate(int $size, int $alignment = self::DEFAULT_ALIGNMENT): int; + + /** + * Whether the blocks handed out stay owned by THIS allocator + * + * `true` means the memory belongs to the allocator (an arena, a shared segment): the + * owner reclaims it as a whole and z-engine must never free an individual block - the + * structures built on top refuse their destroy() path instead of guessing an allocator. + * + * `false` means the block was handed over to z-engine's own bookkeeping (the + * tracked-block registry and the persistent allocator behind it), which is what the + * default EngineAllocator does and what every pre-existing caller relies on. + */ + public function ownsAllocations(): bool; +} diff --git a/src/Memory/EngineAllocator.php b/src/Memory/EngineAllocator.php new file mode 100644 index 0000000..5a77a74 --- /dev/null +++ b/src/Memory/EngineAllocator.php @@ -0,0 +1,126 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Memory; + +use ZEngine\Core; + +/** + * The allocator z-engine has always used: raw blocks minted through PHP's FFI allocator + * + * Three modes, one per allocation shape the framework needs. They are exactly the three + * calls the persistent primitives used to hardcode, so passing the matching instance (or + * nothing at all, since every seam defaults to it) reproduces the previous behavior + * byte for byte: + * + * - trackedPersistent() - Core::trackedNew(..., persistent) -> pemalloc(size, 1), i.e. + * plain malloc, recorded in the tracked-block registry so untrackAndFree() may release + * it later through the very allocator that minted it. The allocation class of persistent + * object clones and of persistent hashtable structs. + * - trackedRequest() - Core::trackedNew(..., request) -> emalloc, registry-tracked. The + * allocation class of an ordinary owned HashTable, which dies with the request. + * - persistent() - Core::new(..., owned: false, persistent: true), malloc-backed and + * deliberately NOT tracked: the allocation class of persistent (interned) zend_strings, + * which are immortal by design and are never handed to untrackAndFree(). + * + * Instances are stateless, so each mode is a process-wide singleton; identity comparison + * against them is a legitimate way to ask "is this the default allocator?". + * + * Blocks are handed over to z-engine's own bookkeeping, never kept by the allocator, so + * ownsAllocations() is false: the structures built on top may release them (destroy()). + */ +final class EngineAllocator implements Allocator +{ + /** + * What malloc() guarantees for the persistent modes (alignof(max_align_t)) + */ + private const int MALLOC_ALIGNMENT = 16; + + /** + * What the Zend memory manager guarantees for the request mode (ZEND_MM_ALIGNMENT) + */ + private const int REQUEST_ALIGNMENT = 8; + + /** + * One shared instance per mode, keyed "tracked?persistent?" + * + * @var array + */ + private static array $instances = []; + + private function __construct( + private readonly bool $persistent, + private readonly bool $tracked, + ) {} + + /** + * Malloc-backed blocks recorded in the tracked-block registry (the persistent default) + */ + public static function trackedPersistent(): self + { + return self::$instances['tracked-persistent'] ??= new self(persistent: true, tracked: true); + } + + /** + * Request-lifetime blocks recorded in the tracked-block registry (the plain default) + */ + public static function trackedRequest(): self + { + return self::$instances['tracked-request'] ??= new self(persistent: false, tracked: true); + } + + /** + * Malloc-backed blocks OUTSIDE the tracked-block registry (immortal by design) + */ + public static function persistent(): self + { + return self::$instances['untracked-persistent'] ??= new self(persistent: true, tracked: false); + } + + /** + * @inheritDoc + */ + #[\Override] + public function allocate(int $size, int $alignment = Allocator::DEFAULT_ALIGNMENT): int + { + if ($size <= 0) { + throw AllocationException::invalidSize($size); + } + if ($alignment <= 0 || ($alignment & ($alignment - 1)) !== 0) { + throw AllocationException::invalidAlignment($alignment); + } + $guaranteed = $this->persistent ? self::MALLOC_ALIGNMENT : self::REQUEST_ALIGNMENT; + if ($alignment > $guaranteed) { + throw AllocationException::unsupportedAlignment($alignment, $guaranteed); + } + + // FFI::new() zeroes whatever it allocates, in both allocation classes, which is + // the "block reads as zero" half of the Allocator contract + $block = $this->tracked + ? Core::trackedNew("char[{$size}]", $this->persistent) + : Core::new("char[{$size}]", false, $this->persistent); + + // The registry keys blocks by the address of the buffer itself, so the same number + // is what a later untrackAndFree()/persistentFree() has to be given + return Core::addressOf(Core::addr($block)); + } + + /** + * @inheritDoc + */ + #[\Override] + public function ownsAllocations(): bool + { + return false; + } +} diff --git a/src/Type/HashTable.php b/src/Type/HashTable.php index 5122ef5..d20df71 100644 --- a/src/Type/HashTable.php +++ b/src/Type/HashTable.php @@ -25,6 +25,8 @@ use ZEngine\Generated\zend_function; use ZEngine\Generated\zend_internal_function; use ZEngine\Generated\zend_refcounted_h; +use ZEngine\Memory\Allocator; +use ZEngine\Memory\EngineAllocator; use ZEngine\Reflection\ReflectionValue; /** @@ -96,6 +98,14 @@ class HashTable implements IteratorAggregate, Countable, ReferenceCountedInterfa */ protected object $pointer; + /** + * Whether the struct block belongs to a FOREIGN allocator (an arena, a shared segment) + * + * Set from the allocator's own ownership report at construction time and false for every + * borrowed (fromCData) view: destroy() refuses to free memory z-engine did not allocate. + */ + protected bool $externallyAllocated = false; + /** * Creates a NEW empty engine-compatible hashtable OWNED by this wrapper * @@ -108,11 +118,25 @@ class HashTable implements IteratorAggregate, Countable, ReferenceCountedInterfa * by the debug-build leak gate). * * A BORROWED view over an engine-owned table is a different construction: fromCData(). + * + * @param Allocator|null $allocator Source of the struct block; the default is z-engine's + * own tracked FFI allocator in the allocation class of + * this table (malloc for persistent subclasses, request + * memory otherwise). A foreign allocator (arena, shared + * segment) keeps ownership of the block: such a table + * refuses destroy(), its owner reclaims the memory. */ - public function __construct() + public function __construct(?Allocator $allocator = null) { - $memory = Core::trackedNew(HashTableStruct::class, static::isPersistentAllocation()); - $pointer = Core::cast(HashTableStruct::class, Core::addr($memory)); + $allocator ??= static::defaultAllocator(); + + $address = $allocator->allocate( + Core::sizeOfType(HashTableStruct::class), + Allocator::ENGINE_STRUCT_ALIGNMENT, + ); + $pointer = Core::pointerAtAddress(HashTableStruct::class, $address); + + $this->externallyAllocated = $allocator->ownsAllocations(); $gcHeader = $pointer->gc; $gcInfo = $gcHeader->u; @@ -161,9 +185,15 @@ public static function fromCData(object $hashInstance): static * view. Stored payloads are not touched (pDestructor is NULL by construction), so * whoever wrote a value into the table still owns it. Sealed persistent tables are * handled by the PersistentHashTable override. + * + * A table built on a FOREIGN allocator is refused: both frees below assume z-engine's + * own allocator, and running them over an arena block would corrupt somebody else's + * heap. The arena owner reclaims such a table by dropping the whole region. */ public function destroy(): void { + $this->assertOwnedMemory(); + Core::call('zend_hash_destroy', $this->pointer); // The engine-grown data block is gone; release the struct through the @@ -179,6 +209,27 @@ protected static function isPersistentAllocation(): bool return false; } + /** + * Refuses any release path for memory that belongs to a foreign allocator + */ + protected function assertOwnedMemory(): void + { + if ($this->externallyAllocated) { + throw TypeOperationException::externallyAllocatedTable(); + } + } + + /** + * Allocator used when the constructor is not given one: z-engine's tracked FFI blocks + * in the allocation class this table declares + */ + protected static function defaultAllocator(): Allocator + { + return static::isPersistentAllocation() + ? EngineAllocator::trackedPersistent() + : EngineAllocator::trackedRequest(); + } + /** * GC header for the constructor: an ordinary collectable request array */ diff --git a/src/Type/PersistentHashTable.php b/src/Type/PersistentHashTable.php index 2b691ad..8eba318 100644 --- a/src/Type/PersistentHashTable.php +++ b/src/Type/PersistentHashTable.php @@ -14,6 +14,8 @@ namespace ZEngine\Type; use ZEngine\Core; +use ZEngine\Generated\Bucket; +use ZEngine\Memory\Allocator; use ZEngine\Reflection\ReflectionValue; /** @@ -46,6 +48,31 @@ */ final class PersistentHashTable extends HashTable { + /** + * Number of uint32_t hash slots the engine reserves per bucket (HT_SIZE_TO_MASK) + * + * @see zend_types.h:HT_SIZE_TO_MASK - the mask is -(nTableSize + nTableSize), so the + * hash part holds twice as many slots as the table has buckets + */ + private const int HASH_SLOTS_PER_BUCKET = 2; + + /** + * Byte pattern of an empty hash slot: HT_INVALID_IDX is 0xFFFFFFFF, so the engine's + * own HT_HASH_RESET is a memset of 0xFF over the whole hash part + */ + private const string EMPTY_HASH_SLOT_BYTE = "\xFF"; + + /** + * Address of the externally allocated arData block, or null while the table uses + * engine-allocated storage (the shared sentinel included) + */ + private ?int $externalStorageAddress = null; + + /** + * Number of buckets the installed external block was sized for (0 without one) + */ + private int $externalCapacity = 0; + /** * Allocation class for the inherited constructor: malloc-backed, outlives the request */ @@ -68,6 +95,171 @@ protected static function gcTypeInfo(): int | Core::engineConstant('GC_NOT_COLLECTABLE'); } + /** + * Mints a table whose bucket storage lives in memory the CALLER allocated + * + * The pairing of the two seams: the struct comes from $allocator (an arena, say), the + * arData block from an address the caller sized with externalStorageSize(). Because + * the block is installed before the first insert, the engine never real-initializes + * (and therefore never pemallocs) storage for this table at all. + * + * @param int $address Address of a zeroed block of externalStorageSize($capacity) bytes + * @param int $capacity Number of buckets the block was sized for + * @param Allocator|null $allocator Source of the struct block (see the constructor) + */ + public static function withExternalStorage(int $address, int $capacity, ?Allocator $allocator = null): self + { + $table = new self($allocator); + $table->installExternalStorage($address, $capacity); + + return $table; + } + + /** + * Byte size of the arData block a table of $capacity buckets needs + * + * The engine's own HT_SIZE_EX(nTableSize, HT_SIZE_TO_MASK(nTableSize)): a hash part of + * two uint32_t slots per bucket, immediately followed by the Bucket area. Both parts + * are one allocation, which is why arData points INTO it (past the hash part) rather + * than at its start. + * + * @see zend_types.h:HT_SIZE_EX/HT_HASH_SIZE/HT_DATA_SIZE + */ + public static function externalStorageSize(int $capacity): int + { + self::assertValidCapacity($capacity); + + return self::hashPartSize($capacity) + $capacity * Core::sizeOfType(Bucket::class); + } + + /** + * Installs a pre-sized, externally allocated arData block on an untouched table + * + * Port of zend_hash.c:zend_hash_real_init_mixed() with the pemalloc replaced by the + * caller's block: nTableSize/nTableMask are set for $capacity, the hash part is reset + * to HT_INVALID_IDX, arData is pointed past it and HASH_FLAG_UNINITIALIZED is cleared + * (HASH_FLAG_STATIC_KEYS takes its place, exactly as the engine does - persistent + * interned keys satisfy it, and the engine clears the flag itself if a non-interned + * key ever arrives). The iterator count in the high flag bytes is left alone. + * + * Required state: the table must still be UNINITIALIZED, i.e. nothing has been inserted + * yet - once the engine has real-initialized a table, its storage is engine-owned and + * replacing it would leak (or double-free) that block. + * + * GROWTH IS THE HAZARD. The engine grows a full table by perealloc()ing HT_GET_DATA_ADDR, + * which for an external block means a realloc of memory the process heap knows nothing + * about. There is no engine hook to forbid that, so the guard is twofold: + * + * - every insert THROUGH THIS WRAPPER checks the remaining capacity first and refuses + * the write that would trigger the resize, so growth never starts; + * - assertNoGrowth() verifies after the fact that arData and the capacity are still the + * installed ones, for the paths (engine C code writing into the same table) that this + * class cannot intercept. + * + * @param int $address Address of a block of externalStorageSize($capacity) zeroed bytes + * @param int $capacity Number of buckets the block was sized for, a power of two + */ + public function installExternalStorage(int $address, int $capacity): void + { + if ($address === 0) { + throw TypeOperationException::invalidStorageAddress(); + } + self::assertValidCapacity($capacity); + + $isUninitialized = ($this->pointer->u->flags & Core::engineConstant('HASH_FLAG_UNINITIALIZED')) !== 0; + if (!$isUninitialized || $this->pointer->nNumUsed !== 0) { + throw TypeOperationException::storageAlreadyInitialized(); + } + + // HT_HASH_RESET: the whole hash part reads as HT_INVALID_IDX (0xFFFFFFFF) before + // the first lookup walks it + $hashPartSize = self::hashPartSize($capacity); + Core::memcpy( + Core::pointerAtAddress('char *', $address), + str_repeat(self::EMPTY_HASH_SLOT_BYTE, $hashPartSize), + $hashPartSize, + ); + + $this->pointer->nTableSize = $capacity; + $this->pointer->nTableMask = self::maskFor($capacity); + // HT_SET_DATA_ADDR: the buckets start right behind the hash part of the same block + $this->pointer->arData = Core::pointerAtAddress(Bucket::class, $address + $hashPartSize); + // Only the flags BYTE, so the iterator count in the neighbouring bytes survives - + // the very reason the engine writes u.v.flags here instead of the whole word + $this->pointer->u->v->flags = Core::engineConstant('HASH_FLAG_STATIC_KEYS'); + $this->pointer->nNumUsed = 0; + $this->pointer->nNumOfElements = 0; + + $this->externalStorageAddress = $address; + $this->externalCapacity = $capacity; + } + + /** + * Whether this table stores its buckets in externally allocated memory + */ + public function hasExternalStorage(): bool + { + return $this->externalStorageAddress !== null; + } + + /** + * Address of the installed external block, or null when the storage is engine-allocated + */ + public function getExternalStorageAddress(): ?int + { + return $this->externalStorageAddress; + } + + /** + * Number of buckets the installed external block was sized for (0 without one) + */ + public function getCapacity(): int + { + return $this->externalCapacity; + } + + /** + * Number of bucket slots still free in the installed external block + * + * Counts SLOTS, not elements: a deleted bucket keeps its slot until a rehash reclaims + * it, and the engine's resize decision is made on nNumUsed for exactly that reason. + * Meaningless (and reported as zero) without external storage. + */ + public function getRemainingCapacity(): int + { + if ($this->externalStorageAddress === null) { + return 0; + } + + return max($this->externalCapacity - $this->pointer->nNumUsed, 0); + } + + /** + * Verifies that the engine has not moved or resized the installed bucket storage + * + * The after-the-fact half of the growth guard: a table backed by external memory must + * still point at the very block that was installed, with the very capacity it was sized + * for. A mismatch means engine code grew the table behind this wrapper's back - the + * external block has been perealloc()ed and the arena is already inconsistent, so the + * exception is a diagnosis, not a rescue. + * + * A no-op for tables whose storage is engine-allocated: there is nothing to protect. + */ + public function assertNoGrowth(): void + { + if ($this->externalStorageAddress === null) { + return; + } + $arData = $this->pointer->arData; + // An initialized table always carries a data block + assert($arData !== null); + + $expected = $this->externalStorageAddress + self::hashPartSize($this->externalCapacity); + if (Core::addressOf($arData) !== $expected || $this->pointer->nTableSize !== $this->externalCapacity) { + throw TypeOperationException::storageRelocated(); + } + } + /** * Upserts a value under a persistent interned string key * @@ -95,6 +287,8 @@ public function add(string $key, ReflectionValue $value): void */ public function addInterned(StringEntry $key, ReflectionValue $value): void { + $this->assertSlotAvailable(fn(): bool => $this->find($key->getStringValue()) !== null); + $result = Core::call( 'zend_hash_add_or_update', $this->pointer, @@ -105,6 +299,7 @@ public function addInterned(StringEntry $key, ReflectionValue $value): void if ($result === null) { throw TypeOperationException::cannotStoreKey($key->getStringValue()); } + $this->assertNoGrowth(); } /** @@ -113,6 +308,8 @@ public function addInterned(StringEntry $key, ReflectionValue $value): void #[\Override] public function addIndex(int $key, ReflectionValue $value): void { + $this->assertSlotAvailable(fn(): bool => $this->findIndex($key) !== null); + $result = Core::call( 'zend_hash_index_add_or_update', $this->pointer, @@ -123,6 +320,7 @@ public function addIndex(int $key, ReflectionValue $value): void if ($result === null) { throw TypeOperationException::cannotStoreIndex($key); } + $this->assertNoGrowth(); } /** @@ -166,6 +364,8 @@ public function markImmutable(): void #[\Override] public function destroy(): void { + $this->assertOwnedMemory(); + // Sealed tables sit at the immutable refcount of 2; the engine asserts <= 1 $this->pointer->gc->refcount = 1; @@ -178,4 +378,74 @@ public function destroy(): void Core::persistentFree($this->pointer); } + /** + * Refuses every release path once the table lives in memory z-engine does not own + * + * Both frees of destroy() assume z-engine's persistent allocator: zend_hash_destroy() + * pefree()s HT_GET_DATA_ADDR (the installed external block) and the struct itself goes + * through free(3). Either one over arena memory corrupts the owner's bookkeeping, so + * such a table is released by dropping the region it lives in, never here. + */ + #[\Override] + protected function assertOwnedMemory(): void + { + parent::assertOwnedMemory(); + + if ($this->externalStorageAddress !== null) { + throw TypeOperationException::externalStorageInstalled(); + } + } + + /** + * Refuses an insert that would make the engine grow externally allocated storage + * + * The engine resizes as soon as an insert finds nNumUsed == nTableSize, so the write + * that fills the last slot is still fine and only the NEXT one has to be stopped. An + * upsert of a key that is already there consumes no slot at all, which is why the + * existence probe is passed lazily: it only runs at the capacity boundary. + * + * @param callable(): bool $replacesExistingBucket Whether the pending write is an upsert + */ + private function assertSlotAvailable(callable $replacesExistingBucket): void + { + if ($this->externalStorageAddress === null || $this->pointer->nNumUsed < $this->externalCapacity) { + return; + } + if ($replacesExistingBucket()) { + return; + } + + throw TypeOperationException::storageCapacityExhausted($this->externalCapacity); + } + + /** + * Byte size of the hash part in front of the buckets (HT_HASH_SIZE of the table's mask) + */ + private static function hashPartSize(int $capacity): int + { + return $capacity * self::HASH_SLOTS_PER_BUCKET * Core::sizeOfType('uint32_t'); + } + + /** + * The engine's HT_SIZE_TO_MASK(nTableSize): -(2 * capacity), as an uint32_t + */ + private static function maskFor(int $capacity): int + { + return (-($capacity + $capacity)) & 0xFFFFFFFF; + } + + /** + * A capacity the engine can address: a power of two, no smaller than HT_MIN_SIZE + * + * Mirrors zend_hash.c:zend_hash_check_size(), except that it refuses instead of + * rounding up - an external block was sized by the caller, and silently pretending it + * holds more buckets than it does is exactly the corruption this API exists to prevent. + */ + private static function assertValidCapacity(int $capacity): void + { + $minimalCapacity = Core::engineConstant('HT_MIN_SIZE'); + if ($capacity < $minimalCapacity || ($capacity & ($capacity - 1)) !== 0) { + throw TypeOperationException::invalidStorageCapacity($capacity, $minimalCapacity); + } + } } diff --git a/src/Type/TypeOperationException.php b/src/Type/TypeOperationException.php index 81b2766..f5470b0 100644 --- a/src/Type/TypeOperationException.php +++ b/src/Type/TypeOperationException.php @@ -62,6 +62,79 @@ public static function functionNotPublished(string $key): self return new self("Function {$key} was not published in the table"); } + /** + * Raised when a release path is asked to free memory a foreign allocator owns + */ + public static function externallyAllocatedTable(): self + { + return new self( + 'This hashtable was built on a foreign allocator: its memory belongs to that ' + . 'allocator and only its owner may release it', + ); + } + + /** + * Raised when a release path is asked to free externally installed bucket storage + */ + public static function externalStorageInstalled(): self + { + return new self( + 'This hashtable stores its buckets in externally allocated memory: releasing it ' + . 'here would free a block z-engine does not own', + ); + } + + /** + * Raised when external storage is installed on a table the engine has already initialized + */ + public static function storageAlreadyInitialized(): self + { + return new self( + 'External bucket storage can only be installed on an untouched table, before the ' + . 'first insert makes the engine allocate storage of its own', + ); + } + + /** + * Raised when the address of an external bucket storage block is zero + */ + public static function invalidStorageAddress(): self + { + return new self('The address of an external bucket storage block must not be zero'); + } + + /** + * Raised when a bucket capacity is not one the engine can address + */ + public static function invalidStorageCapacity(int $capacity, int $minimalCapacity): self + { + return new self( + "A bucket capacity must be a power of two of at least {$minimalCapacity}, {$capacity} given", + ); + } + + /** + * Raised when an insert would make the engine grow externally allocated storage + */ + public static function storageCapacityExhausted(int $capacity): self + { + return new self( + "All {$capacity} bucket slots of the external storage are used: another insert would " + . 'make the engine reallocate a block it does not own', + ); + } + + /** + * Raised when the engine has moved the bucket storage of an externally backed table + */ + public static function storageRelocated(): self + { + return new self( + 'The engine has reallocated the bucket storage of a table backed by external memory: ' + . 'the installed block is no longer the one in use', + ); + } + public static function danglingObjectEntry(): self { return new self('The underlying object has been destroyed, this entry is dangling'); From 9b3f226e3613ae305207f3d3c3aea61e0a8d586d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 19:18:06 +0000 Subject: [PATCH 2/5] feat(memory): thread the allocator through the persistent primitives The three primitives every persistent graph is built from - the object clones of PersistentObjectFactory, the interned blocks of StringEntry::persistent(Interned) and the tables of PersistentGraphCloner - now take an optional allocator and pass it on, so a whole graph can be cloned into memory the caller owns. Passing nothing keeps each primitive on its own historical default, which is deliberately not the same allocator for all of them: object clones and table structs stay tracked malloc blocks, string blocks stay untracked ones, exactly as before. Core::offsetOfField() is the named form of the type(...)->getStructFieldOffset(...) pair the string minting needed - the same remedy sizeOfType() is for sizeof(type(...)), and one raw FFI\CType less in the API surface. Covered by an arena-shaped test allocator (bump pointer over one owned region, reporting ownership): clones, strings and whole cyclic graphs land inside the region, nothing of them reaches z-engine's block registry, and a table built on it refuses to be freed. The external bucket storage is covered end to end - layout math against HT_SIZE_EX, lookups through the installed hash part, the capacity guard at the resize boundary, upserts still allowed when full, and the relocation diagnosis. Refs #223 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B9xaBchjdo1atarNZ6sZqe --- docs/persistent-heap.md | 43 +++ src/Core.php | 17 + src/Memory/PersistentGraphCloner.php | 19 +- src/Type/PersistentObjectFactory.php | 20 +- src/Type/StringEntry.php | 24 +- tests/CoreTypedEntryPointsTest.php | 19 ++ tests/Memory/AllocatorSeamTest.php | 147 +++++++++ tests/Memory/EngineAllocatorTest.php | 123 ++++++++ tests/Stub/RecordingArenaAllocator.php | 105 +++++++ ...PersistentHashTableExternalStorageTest.php | 294 ++++++++++++++++++ 10 files changed, 795 insertions(+), 16 deletions(-) create mode 100644 tests/Memory/AllocatorSeamTest.php create mode 100644 tests/Memory/EngineAllocatorTest.php create mode 100644 tests/Stub/RecordingArenaAllocator.php create mode 100644 tests/Type/PersistentHashTableExternalStorageTest.php diff --git a/docs/persistent-heap.md b/docs/persistent-heap.md index 4db3fa8..edda1aa 100644 --- a/docs/persistent-heap.md +++ b/docs/persistent-heap.md @@ -104,6 +104,49 @@ Persistent strings are interned-style (immutable, non-refcounted), persistent ar sealed immutable tables — both are copied into zvals without refcounting and any userland mutation copy-on-writes into request memory, leaving the persistent block untouched. +## Where the persistent bytes come from (the allocator seam) + +Every persistent block the framework mints — an object clone, an interned string block, a +hashtable struct — is allocated through a `ZEngine\Memory\Allocator`. The default is +`EngineAllocator`, which reproduces the three shapes z-engine used to hardcode (tracked +malloc, tracked request memory, untracked malloc), so a caller that passes nothing sees no +change at all. + +Passing one is what lets a graph live somewhere other than the process heap — a +fork-shared mmap arena, a shared-memory segment: + +```php +$graph = (new PersistentGraphCloner($arena))->persist($root); // objects + strings + tables +$clone = PersistentObjectFactory::persistentClone($rawObject, $arena); // one object +$block = StringEntry::persistentInterned('key', $arena); // one string +$table = new PersistentHashTable($arena); // one table struct +``` + +The interface speaks in **addresses**, never `FFI\CData` (AGENTS.md): an implementation in +a consumer package binds its own `mmap`/`shm` primitives with `FFI::cdef` and returns the +integer it computed. It also reports whether it keeps ownership of what it hands out — +`ownsAllocations()`. A structure built on such memory refuses `destroy()`: both frees +assume z-engine's own allocator, and the arena owner releases the region as a whole. + +A table's struct is only half its memory; the buckets are the other half, and the engine +would `pemalloc` them on the first insert. `PersistentHashTable::installExternalStorage()` +takes that half too: + +```php +$capacity = 1024; // a power of two +$address = $arena->allocate(PersistentHashTable::externalStorageSize($capacity)); +$table = PersistentHashTable::withExternalStorage($address, $capacity, $arena); +``` + +The installed block carries the engine's own `HT_SIZE_EX` layout (two `uint32_t` hash slots +per bucket, reset to `HT_INVALID_IDX`, followed by the `Bucket` area), and installation is +only allowed **before the first insert**, so the engine never allocates storage of its own. +Growth is the hazard afterwards — the engine grows a full table by `perealloc`ing exactly +that block — and is guarded from both sides: inserts through the wrapper refuse the write +that would trigger the resize (`getRemainingCapacity()` reports the headroom), and +`assertNoGrowth()` diagnoses a relocation caused by engine paths the wrapper cannot +intercept. + ## Storage layout and the anchor Everything the heap needs across requests lives in engine-visible persistent memory — diff --git a/src/Core.php b/src/Core.php index da5bdc2..3f51c38 100644 --- a/src/Core.php +++ b/src/Core.php @@ -1022,6 +1022,23 @@ public static function free(object $variable): void FFI::free(self::toCData($variable)); } + /** + * Returns the byte offset of a field inside an engine structure + * + * The named form of the type(...)->getStructFieldOffset(...) pair, and the one consumers + * should use: it answers "where do the inline properties start in a zend_object?" without + * a raw FFI\CType ever crossing the API boundary - the same remedy sizeOfType() is for + * sizeof(type(...)). + * + * @param class-string|string $type Name of the engine type (eg "zend_object") or a + * generated struct stub class + * @param string $field Name of the declared field + */ + public static function offsetOfField(string $type, string $field): int + { + return self::$engine->type(self::resolveCName($type))->getStructFieldOffset($field); + } + /** * Returns a CType definition for engine by type name * diff --git a/src/Memory/PersistentGraphCloner.php b/src/Memory/PersistentGraphCloner.php index 5427cb4..127fe67 100644 --- a/src/Memory/PersistentGraphCloner.php +++ b/src/Memory/PersistentGraphCloner.php @@ -18,6 +18,7 @@ use ZEngine\Generated\HashTable as HashTableStruct; use ZEngine\Generated\zend_array; use ZEngine\Generated\zend_object; +use ZEngine\Generated\zend_string; use ZEngine\Generated\zval; use ZEngine\Reflection\ReflectionClass; use ZEngine\Reflection\ReflectionValue; @@ -115,7 +116,15 @@ final class PersistentGraphCloner */ private readonly int $objectTypeFlags; - public function __construct() + /** + * @param Allocator|null $allocator Source of every persistent byte this cloner mints - + * object clones, interned string blocks and table + * structs alike. Null keeps each primitive on its own + * historical default (z-engine's malloc-backed FFI + * allocator), which is not the same allocator for all + * three: strings are minted untracked, the rest tracked. + */ + public function __construct(private readonly ?Allocator $allocator = null) { $refcounted = Core::engineConstant('IS_TYPE_REFCOUNTED') | Core::engineConstant('IS_TYPE_COLLECTABLE'); @@ -273,7 +282,7 @@ private function cloneObject(object $sourceObject): object return $this->objectMap[$address]; } - $clone = PersistentObjectFactory::persistentClone($sourceObject); + $clone = PersistentObjectFactory::persistentClone($sourceObject, $this->allocator); $cloneEntry = ObjectEntry::fromCData($clone); // The byte-copied handle belongs to the source in the CURRENT request; the clone // receives a fresh handle at every re-attachment (ObjectStore::put) @@ -370,7 +379,7 @@ private function cloneArray(object $sourceArray): object return $this->arrayMap[$address]; } - $table = new PersistentHashTable(); + $table = new PersistentHashTable($this->allocator); $rawTable = $table->getRawValue(); // Record the mapping before filling: an element may reach this very array again @@ -411,11 +420,11 @@ private function persistString(string $content): StringEntry return $this->stringPool[$content]; } - $entry = StringEntry::persistentInterned($content); + $entry = StringEntry::persistentInterned($content, $this->allocator); $this->stringPool[$content] = $entry; $this->strings[] = $entry->getRawValue(); - $this->bytes += Core::type('zend_string')->getStructFieldOffset('val') + strlen($content) + 1; + $this->bytes += Core::offsetOfField(zend_string::class, 'val') + strlen($content) + 1; return $entry; } diff --git a/src/Type/PersistentObjectFactory.php b/src/Type/PersistentObjectFactory.php index c3ce180..c78b16b 100644 --- a/src/Type/PersistentObjectFactory.php +++ b/src/Type/PersistentObjectFactory.php @@ -17,6 +17,8 @@ use ZEngine\Core; use ZEngine\Generated\zend_object; use ZEngine\Generated\zend_object_handlers; +use ZEngine\Memory\Allocator; +use ZEngine\Memory\EngineAllocator; use ZEngine\Reflection\ReflectionClass; /** @@ -63,10 +65,17 @@ final class PersistentObjectFactory * Creates a persistent byte-clone of a live zend_object * * @param CData|zend_object $sourceObject zend_object* to clone (must use std_object_handlers) + * @param Allocator|null $allocator Source of the clone's memory; the default is + * z-engine's tracked malloc-backed allocator, ie + * exactly the process-heap block this factory has + * always minted. A foreign allocator (a fork-shared + * arena, say) puts the clone into ITS memory, and + * the caller releases it the same way it does the + * rest of that region. * * @return zend_object zend_object* in persistent memory, not yet registered in the store */ - public static function persistentClone(object $sourceObject): object + public static function persistentClone(object $sourceObject, ?Allocator $allocator = null): object { /** @var zend_object $source Narrowed to the stub view at the owning boundary */ $source = $sourceObject; @@ -74,10 +83,13 @@ public static function persistentClone(object $sourceObject): object // Engine invariant: every live object carries its class entry assert($sourceClass !== null); $totalSize = ReflectionClass::getObjectSize($sourceClass); - $memory = Core::trackedNew("char[{$totalSize}]", true); - $object = Core::cast(zend_object::class, $memory); + $allocator ??= EngineAllocator::trackedPersistent(); + $object = Core::pointerAtAddress( + zend_object::class, + $allocator->allocate($totalSize, Allocator::ENGINE_STRUCT_ALIGNMENT), + ); - Core::memcpy($memory, Core::cast('char *', $sourceObject), $totalSize); + Core::memcpy($object, Core::cast('char *', $sourceObject), $totalSize); $object->gc->refcount = self::PIN_BASELINE; $object->gc->u->type_info = Core::engineConstant('GC_OBJECT') diff --git a/src/Type/StringEntry.php b/src/Type/StringEntry.php index 30943b6..59dac5b 100644 --- a/src/Type/StringEntry.php +++ b/src/Type/StringEntry.php @@ -18,6 +18,8 @@ use ZEngine\Core; use ZEngine\Generated\zend_refcounted_h; use ZEngine\Generated\zend_string; +use ZEngine\Memory\Allocator; +use ZEngine\Memory\EngineAllocator; use ZEngine\Reflection\ReflectionValue; /** @@ -130,19 +132,25 @@ public static function fromString(string $value): StringEntry * (internal classes and their members), which the engine releases with the persistent * allocator. The struct is built manually because no persistent string constructor is * exported; the layout is verified at boot by the engine layout checks. + * + * @param Allocator|null $allocator Source of the string block; the default is z-engine's + * untracked malloc-backed allocator, ie the immortal + * block this method has always minted. A foreign + * allocator puts the string into ITS memory instead. */ - public static function persistent(string $value): StringEntry + public static function persistent(string $value, ?Allocator $allocator = null): StringEntry { $length = strlen($value); - $valOffset = Core::type(zend_string::class)->getStructFieldOffset('val'); + $valOffset = Core::offsetOfField(zend_string::class, 'val'); - $buffer = Core::new('char[' . ($valOffset + $length + 1) . ']', false, true); - $string = Core::cast(zend_string::class, $buffer); + $allocator ??= EngineAllocator::persistent(); + $address = $allocator->allocate($valOffset + $length + 1, Allocator::ENGINE_STRUCT_ALIGNMENT); + $string = Core::pointerAtAddress(zend_string::class, $address); $string->gc->refcount = 1; $string->gc->u->type_info = Core::engineConstant('GC_STRING') | Core::engineConstant('GC_PERSISTENT'); $string->len = $length; - $valPointer = Core::cast('char *', $buffer) + $valOffset; + $valPointer = Core::pointerAtAddress('char *', $address + $valOffset); assert($valPointer instanceof CData); Core::memcpy($valPointer, $value . "\0", $length + 1); $hash = Core::call('zend_string_hash_func', $string); @@ -168,10 +176,12 @@ public static function persistent(string $value): StringEntry * The string is never registered in the engine's interned tables, so equality * against a real interned string falls back to a content compare - same contract * as opcache SHM strings in processes that attach without the interning pass. + * + * @param Allocator|null $allocator Source of the string block (see persistent()) */ - public static function persistentInterned(string $value): StringEntry + public static function persistentInterned(string $value, ?Allocator $allocator = null): StringEntry { - $stringEntry = static::persistent($value); + $stringEntry = static::persistent($value, $allocator); $pointer = $stringEntry->pointer; $pointer->gc->u->type_info |= Core::engineConstant('GC_IMMUTABLE'); diff --git a/tests/CoreTypedEntryPointsTest.php b/tests/CoreTypedEntryPointsTest.php index ee8c555..ce555b1 100644 --- a/tests/CoreTypedEntryPointsTest.php +++ b/tests/CoreTypedEntryPointsTest.php @@ -56,6 +56,25 @@ public function testStubClassPointerAtAddressIsAPointerCast(): void $this->assertSame($address, Core::addressOf(Core::pointerAtAddress(zval::class, $address))); } + public function testFieldOffsetsAreReachableWithoutATypeHandle(): void + { + // The named replacement for type(...)->getStructFieldOffset(...): a consumer must be + // able to ask where a field starts without a raw FFI\CType crossing the boundary + $this->assertSame( + Core::type('zend_string')->getStructFieldOffset('val'), + Core::offsetOfField(zend_string::class, 'val'), + ); + $this->assertSame( + Core::offsetOfField('zend_string', 'val'), + Core::offsetOfField(zend_string::class, 'val'), + ); + + // The first field of a struct sits at offset zero, the next ones behind it + $this->assertSame(0, Core::offsetOfField(zend_string::class, 'gc')); + $this->assertGreaterThan(0, Core::offsetOfField(zend_string::class, 'len')); + $this->assertLessThan(Core::sizeOfType(zend_string::class), Core::offsetOfField(zend_string::class, 'len')); + } + public function testStubClassesAreNeverLoadedAtRuntime(): void { $this->assertFalse(class_exists(zval::class, false), 'stub classes must stay analysis-only'); diff --git a/tests/Memory/AllocatorSeamTest.php b/tests/Memory/AllocatorSeamTest.php new file mode 100644 index 0000000..32a477c --- /dev/null +++ b/tests/Memory/AllocatorSeamTest.php @@ -0,0 +1,147 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Memory; + +use PHPUnit\Framework\TestCase; +use ZEngine\Core; +use ZEngine\Generated\HashTable as HashTableStruct; +use ZEngine\Generated\zend_string; +use ZEngine\Reflection\ReflectionValue; +use ZEngine\Stub\RecordingArenaAllocator; +use ZEngine\Stub\TestGraphNode; +use ZEngine\Stub\TestPersistentCandidate; +use ZEngine\Type\ObjectEntry; +use ZEngine\Type\PersistentHashTable; +use ZEngine\Type\PersistentObjectFactory; +use ZEngine\Type\StringEntry; +use ZEngine\Type\TypeOperationException; + +/** + * The allocator seam seen from the primitives: every persistent block a caller can redirect + * + * The arena stands in for the fork-shared mmap region of the downstream consumer: it owns its + * memory, so a structure built on it must never be released through z-engine's own allocator. + */ +class AllocatorSeamTest extends TestCase +{ + public function testPersistentCloneCanBeMintedInsideAForeignRegion(): void + { + $arena = new RecordingArenaAllocator(); + $source = new TestPersistentCandidate(); + $source->counter = 42; + + $sourceValue = new ReflectionValue($source); + $clone = PersistentObjectFactory::persistentClone($sourceValue->getRawObject(), $arena); + $sourceValue->release(); + + $this->assertTrue($arena->holds($clone), 'The clone was not allocated from the arena'); + $this->assertCount(1, $arena->allocations); + $this->assertSame(Allocator::ENGINE_STRUCT_ALIGNMENT, $arena->allocations[0]['alignment']); + + // The clone is a full persistent clone, not a degraded one: same header surgery + $entry = ObjectEntry::fromCData($clone); + $this->assertSame(PersistentObjectFactory::PIN_BASELINE, $entry->getReferenceCount()); + $this->assertTrue($entry->isPersistent()); + $this->assertTrue(PersistentObjectFactory::usesStandardHandlers($clone)); + $entry->getPropertySlot(0)->getNativeValue($counter); + $this->assertSame(42, $counter); + } + + public function testPersistentStringsCanBeMintedInsideAForeignRegion(): void + { + $arena = new RecordingArenaAllocator(); + + $string = StringEntry::persistent('arena resident', $arena); + $this->assertTrue($arena->holds($string->getRawValue())); + $this->assertSame('arena resident', $string->getStringValue()); + $this->assertTrue($string->isPersistent()); + $this->assertFalse($string->isInterned()); + + $interned = StringEntry::persistentInterned('arena interned', $arena); + $this->assertTrue($arena->holds($interned->getRawValue())); + $this->assertSame('arena interned', $interned->getStringValue()); + $this->assertTrue($interned->isInterned()); + $this->assertSame(2, $interned->getReferenceCount()); + + // Both blocks are exactly as large as the engine layout requires + $expectedSize = Core::offsetOfField(zend_string::class, 'val') + strlen('arena resident') + 1; + $this->assertSame($expectedSize, $arena->allocations[0]['size']); + } + + public function testHashTableStructCanBeMintedInsideAForeignRegionAndRefusesToBeFreed(): void + { + $arena = new RecordingArenaAllocator(); + $table = new PersistentHashTable($arena); + + $this->assertTrue($arena->holds($table->getRawValue())); + $this->assertSame(Core::sizeOfType(HashTableStruct::class), $arena->allocations[0]['size']); + $this->assertTrue($table->isPersistent()); + + // Releasing arena memory through the framework's own allocator is the corruption + // this ownership report exists to prevent + $this->expectException(TypeOperationException::class); + $this->expectExceptionMessageMatches('/foreign allocator/'); + $table->destroy(); + } + + public function testWholeGraphsAreClonedIntoTheAllocatorTheClonerWasGiven(): void + { + $arena = new RecordingArenaAllocator(); + + $root = new TestGraphNode(); + $root->name = 'root'; + $root->rank = 1; + $child = new TestGraphNode(); + $child->name = 'child'; + $child->rank = 2; + // A cycle, so the identity map is exercised while the arena is in play + $child->parent = $root; + $root->left = $child; + + $graph = (new PersistentGraphCloner($arena))->persist($root); + + $this->assertCount(2, $graph->objects); + foreach ($graph->objects as $object) { + $this->assertTrue($arena->holds($object), 'A cloned object escaped the arena'); + } + foreach ($graph->strings as $string) { + $this->assertTrue($arena->holds($string), 'A cloned string escaped the arena'); + } + $this->assertGreaterThan(0, $arena->allocatedBytes()); + + // Nothing of the graph ended up in z-engine's own block registry + foreach ($graph->objects as $object) { + $this->assertFalse(Core::isTrackedBlock($object)); + } + } + + public function testDefaultsKeepTheHistoricalAllocationClasses(): void + { + // The seam must be invisible to every existing caller: object clones and table + // structs stay tracked malloc blocks, string blocks stay untracked malloc blocks + $source = new TestPersistentCandidate(); + $sourceValue = new ReflectionValue($source); + $clone = PersistentObjectFactory::persistentClone($sourceValue->getRawObject()); + $sourceValue->release(); + $this->assertTrue(Core::isTrackedBlock($clone)); + + $table = new PersistentHashTable(); + $this->assertTrue(Core::isTrackedBlock($table->getRawValue())); + $table->destroy(); + + $string = StringEntry::persistent('untracked by design'); + $this->assertFalse(Core::isTrackedBlock($string->getRawValue())); + $this->assertTrue($string->isPersistent()); + } +} diff --git a/tests/Memory/EngineAllocatorTest.php b/tests/Memory/EngineAllocatorTest.php new file mode 100644 index 0000000..9794ee3 --- /dev/null +++ b/tests/Memory/EngineAllocatorTest.php @@ -0,0 +1,123 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Memory; + +use PHPUnit\Framework\TestCase; +use ZEngine\Core; + +class EngineAllocatorTest extends TestCase +{ + public function testModesAreSharedStatelessInstances(): void + { + $this->assertSame(EngineAllocator::trackedPersistent(), EngineAllocator::trackedPersistent()); + $this->assertSame(EngineAllocator::trackedRequest(), EngineAllocator::trackedRequest()); + $this->assertSame(EngineAllocator::persistent(), EngineAllocator::persistent()); + + $this->assertNotSame(EngineAllocator::trackedPersistent(), EngineAllocator::persistent()); + $this->assertNotSame(EngineAllocator::trackedPersistent(), EngineAllocator::trackedRequest()); + } + + public function testEveryModeHandsOutZeroedAlignedMemory(): void + { + foreach ($this->allModes() as $mode => $allocator) { + $address = $allocator->allocate(128, Allocator::ENGINE_STRUCT_ALIGNMENT); + + $this->assertNotSame(0, $address, "{$mode} returned a null address"); + $this->assertSame(0, $address % Allocator::ENGINE_STRUCT_ALIGNMENT, "{$mode} is misaligned"); + + $block = Core::pointerAtAddress('char *', $address); + for ($offset = 0; $offset < 128; $offset++) { + $byte = $block[$offset]; + $this->assertIsString($byte); + $this->assertSame(0, ord($byte), "{$mode} left byte {$offset} dirty"); + } + } + } + + public function testBlocksAreHandedOverToTheFrameworkNotKeptByTheAllocator(): void + { + foreach ($this->allModes() as $mode => $allocator) { + $this->assertFalse($allocator->ownsAllocations(), "{$mode} claims ownership of its blocks"); + } + } + + public function testOnlyTheTrackedModesEnterTheBlockRegistry(): void + { + // The registry is what untrackAndFree() consults, so the mode a primitive picks + // decides whether the framework may release the block later on + $tracked = Core::pointerAtAddress('char *', EngineAllocator::trackedPersistent()->allocate(64)); + $untracked = Core::pointerAtAddress('char *', EngineAllocator::persistent()->allocate(64)); + + $this->assertTrue(Core::isTrackedBlock($tracked)); + $this->assertFalse(Core::isTrackedBlock($untracked)); + + // The tracked block is releasable through the very allocator that minted it + Core::untrackAndFree($tracked); + $this->assertFalse(Core::isTrackedBlock($tracked)); + } + + public function testDistinctAllocationsNeverOverlap(): void + { + $first = EngineAllocator::trackedPersistent()->allocate(64); + $second = EngineAllocator::trackedPersistent()->allocate(64); + + $this->assertNotSame($first, $second); + $this->assertGreaterThanOrEqual(64, abs($first - $second)); + } + + public function testNonPositiveSizeIsRefused(): void + { + $this->expectException(AllocationException::class); + $this->expectExceptionMessageMatches('/positive number of bytes/'); + + EngineAllocator::trackedPersistent()->allocate(0); + } + + public function testAlignmentThatIsNotAPowerOfTwoIsRefused(): void + { + $this->expectException(AllocationException::class); + $this->expectExceptionMessageMatches('/power of two/'); + + EngineAllocator::trackedPersistent()->allocate(16, 24); + } + + public function testAlignmentBeyondTheAllocatorGuaranteeIsRefused(): void + { + // malloc answers for 16 bytes, the Zend memory manager only for its own alignment: + // an allocator that cannot promise what was asked for says so instead of hoping + $this->expectException(AllocationException::class); + $this->expectExceptionMessageMatches('/guarantees 8-byte alignment/'); + + EngineAllocator::trackedRequest()->allocate(16, 16); + } + + public function testMallocBackedModesSatisfyTheDefaultAlignment(): void + { + $address = EngineAllocator::trackedPersistent()->allocate(32, Allocator::DEFAULT_ALIGNMENT); + + $this->assertSame(0, $address % Allocator::DEFAULT_ALIGNMENT); + } + + /** + * @return array + */ + private function allModes(): array + { + return [ + 'trackedPersistent' => EngineAllocator::trackedPersistent(), + 'trackedRequest' => EngineAllocator::trackedRequest(), + 'persistent' => EngineAllocator::persistent(), + ]; + } +} diff --git a/tests/Stub/RecordingArenaAllocator.php b/tests/Stub/RecordingArenaAllocator.php new file mode 100644 index 0000000..ce88e97 --- /dev/null +++ b/tests/Stub/RecordingArenaAllocator.php @@ -0,0 +1,105 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Stub; + +use ZEngine\Core; +use ZEngine\Memory\AllocationException; +use ZEngine\Memory\Allocator; +use ZEngine\Memory\EngineAllocator; + +/** + * Bump allocator over one owned region: the shape a fork-shared arena has, without the mmap + * + * The region itself is a single malloc block (that part is irrelevant to the seam - what + * matters is that the allocator hands out addresses INSIDE memory it owns and says so), and + * every block is carved out of it by bumping a cursor. Individual blocks are never released: + * the owner drops the whole region, which is exactly why ownsAllocations() is true and why + * structures built on it must refuse their destroy() path. + * + * Every request is recorded so a test can assert what a primitive actually asked for. + */ +final class RecordingArenaAllocator implements Allocator +{ + /** + * Requests served so far, in order + * + * @var list + */ + public private(set) array $allocations = []; + + private readonly int $regionAddress; + + private int $cursor; + + public function __construct(private readonly int $regionSize = 1 << 20) + { + // The region is zeroed once here; the bump cursor never revisits a byte, so every + // block handed out satisfies the "reads as zero" half of the Allocator contract + $this->regionAddress = EngineAllocator::persistent()->allocate($regionSize); + $this->cursor = $this->regionAddress; + } + + #[\Override] + public function allocate(int $size, int $alignment = Allocator::DEFAULT_ALIGNMENT): int + { + if ($size <= 0) { + throw AllocationException::invalidSize($size); + } + if ($alignment <= 0 || ($alignment & ($alignment - 1)) !== 0) { + throw AllocationException::invalidAlignment($alignment); + } + + $address = ($this->cursor + $alignment - 1) & ~($alignment - 1); + if ($address + $size > $this->regionAddress + $this->regionSize) { + throw AllocationException::invalidSize($size); + } + $this->cursor = $address + $size; + + $this->allocations[] = ['size' => $size, 'alignment' => $alignment, 'address' => $address]; + + return $address; + } + + #[\Override] + public function ownsAllocations(): bool + { + return true; + } + + /** + * Whether the given address points into this arena's region + */ + public function contains(int $address): bool + { + return $address >= $this->regionAddress && $address < $this->regionAddress + $this->regionSize; + } + + /** + * Whether the block the given engine pointer refers to lives in this arena's region + * + * @param object $pointer Runtime value is always CData; stub-typed views are accepted + */ + public function holds(object $pointer): bool + { + return $this->contains(Core::addressOf($pointer)); + } + + /** + * Total number of bytes handed out so far + */ + public function allocatedBytes(): int + { + return array_sum(array_column($this->allocations, 'size')); + } +} diff --git a/tests/Type/PersistentHashTableExternalStorageTest.php b/tests/Type/PersistentHashTableExternalStorageTest.php new file mode 100644 index 0000000..288d37a --- /dev/null +++ b/tests/Type/PersistentHashTableExternalStorageTest.php @@ -0,0 +1,294 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Type; + +use PHPUnit\Framework\TestCase; +use ZEngine\Core; +use ZEngine\Generated\Bucket; +use ZEngine\Memory\Allocator; +use ZEngine\Memory\EngineAllocator; +use ZEngine\Reflection\ReflectionValue; +use ZEngine\Stub\RecordingArenaAllocator; + +/** + * Bucket storage that the CALLER allocated: the second half of the arena story + * + * A table whose struct lives in an arena but whose buckets the engine pemallocs is still + * half in the process heap. installExternalStorage() closes that gap - and the growth guard + * is what keeps the engine from silently reopening it with a perealloc. + */ +class PersistentHashTableExternalStorageTest extends TestCase +{ + public function testStorageSizeMatchesTheEngineLayout(): void + { + // HT_SIZE_EX(nTableSize, HT_SIZE_TO_MASK(nTableSize)): two uint32_t hash slots per + // bucket, then the buckets themselves + $bucketSize = Core::sizeOfType(Bucket::class); + foreach ([8, 16, 64, 1024] as $capacity) { + $this->assertSame( + $capacity * 2 * 4 + $capacity * $bucketSize, + PersistentHashTable::externalStorageSize($capacity), + ); + } + } + + public function testCapacityMustBeAPowerOfTwoAboveTheEngineMinimum(): void + { + foreach ([0, 1, 4, 12, 100] as $invalid) { + try { + PersistentHashTable::externalStorageSize($invalid); + $this->fail("Capacity {$invalid} should have been refused"); + } catch (TypeOperationException $exception) { + $this->assertMatchesRegularExpression('/power of two/', $exception->getMessage()); + } + } + + $this->assertGreaterThan(0, PersistentHashTable::externalStorageSize(Core::engineConstant('HT_MIN_SIZE'))); + } + + public function testInstalledStorageBacksEveryBucketOfTheTable(): void + { + $capacity = 16; + $table = $this->tableWithExternalStorage($capacity); + $address = $table->getExternalStorageAddress(); + + $this->assertTrue($table->hasExternalStorage()); + $this->assertNotNull($address); + $this->assertSame($capacity, $table->getCapacity()); + $this->assertSame($capacity, $table->getRemainingCapacity()); + + for ($index = 0; $index < $capacity; $index++) { + $value = new ReflectionValue($index * 3); + $table->add("key{$index}", $value); + $value->release(); + } + + $this->assertCount($capacity, $table); + $this->assertSame(0, $table->getRemainingCapacity()); + + // Both lookup paths walk the installed hash part and bucket area + $this->assertSame(21, self::valueOf($table->find('key7'))); + $this->assertNull($table->find('missing')); + $this->assertSame(range(0, ($capacity - 1) * 3, 3), array_values(array_map( + static function (ReflectionValue $value): int { + $value->getNativeValue($native); + assert(is_int($native)); + + return $native; + }, + iterator_to_array($table->getIterator()), + ))); + + // The engine never moved the block it was given + $table->assertNoGrowth(); + $arData = $table->getRawValue()->arData; + assert($arData !== null); + $this->assertSame($address + $capacity * 2 * 4, Core::addressOf($arData)); + } + + public function testIntegerKeysUseTheInstalledStorageToo(): void + { + $table = $this->tableWithExternalStorage(8); + + foreach ([5, 9, 17] as $key) { + $value = new ReflectionValue($key * 100); + $table->addIndex($key, $value); + $value->release(); + } + + $this->assertSame(900, self::valueOf($table->findIndex(9))); + $this->assertSame(5, $table->getRemainingCapacity()); + $table->assertNoGrowth(); + } + + public function testAnInsertThatWouldGrowTheStorageIsRefused(): void + { + $capacity = 8; + $table = $this->tableWithExternalStorage($capacity); + + for ($index = 0; $index < $capacity; $index++) { + $value = new ReflectionValue($index); + $table->addIndex($index, $value); + $value->release(); + } + + $overflow = new ReflectionValue(999); + try { + $table->addIndex($capacity, $overflow); + $this->fail('The insert that would have made the engine perealloc was not refused'); + } catch (TypeOperationException $exception) { + $this->assertMatchesRegularExpression('/bucket slots/', $exception->getMessage()); + } + // Refused BEFORE the engine saw the write: the table is untouched and intact + $this->assertCount($capacity, $table); + $table->assertNoGrowth(); + + // A string key is guarded on the very same boundary + try { + $table->add('one more', $overflow); + $this->fail('The string-keyed insert past the capacity was not refused'); + } catch (TypeOperationException $exception) { + $this->assertMatchesRegularExpression('/bucket slots/', $exception->getMessage()); + } + $overflow->release(); + $table->assertNoGrowth(); + } + + public function testUpsertingAnExistingKeyStaysAllowedAtFullCapacity(): void + { + $capacity = 8; + $table = $this->tableWithExternalStorage($capacity); + + for ($index = 0; $index < $capacity; $index++) { + $value = new ReflectionValue($index); + $table->add("key{$index}", $value); + $value->release(); + } + $this->assertSame(0, $table->getRemainingCapacity()); + + // An upsert consumes no bucket slot, so the guard must let it through + $replacement = new ReflectionValue(4242); + $table->add('key3', $replacement); + $replacement->release(); + + $this->assertSame(4242, self::valueOf($table->find('key3'))); + $this->assertCount($capacity, $table, 'An upsert must not add an element'); + $table->assertNoGrowth(); + } + + public function testStorageCannotBeInstalledOnATableTheEngineAlreadyInitialized(): void + { + $table = new PersistentHashTable(); + $value = new ReflectionValue(1); + $table->add('first', $value); + $value->release(); + + $capacity = 8; + $address = EngineAllocator::persistent()->allocate( + PersistentHashTable::externalStorageSize($capacity), + Allocator::DEFAULT_ALIGNMENT, + ); + + try { + $table->installExternalStorage($address, $capacity); + $this->fail('Storage was installed over an engine-allocated data block'); + } catch (TypeOperationException $exception) { + $this->assertMatchesRegularExpression('/untouched table/', $exception->getMessage()); + } + + $table->destroy(); + } + + public function testAZeroStorageAddressIsRefused(): void + { + $table = new PersistentHashTable(); + + $this->expectException(TypeOperationException::class); + $this->expectExceptionMessageMatches('/must not be zero/'); + try { + $table->installExternalStorage(0, 8); + } finally { + $table->destroy(); + } + } + + public function testATableBackedByExternalStorageRefusesToBeDestroyed(): void + { + $table = $this->tableWithExternalStorage(8); + + // zend_hash_destroy() would pefree() the caller's block - that is the whole hazard + $this->expectException(TypeOperationException::class); + $this->expectExceptionMessageMatches('/externally allocated memory/'); + $table->destroy(); + } + + public function testAStorageRelocationIsDiagnosed(): void + { + $table = $this->tableWithExternalStorage(8); + $address = $table->getExternalStorageAddress(); + $this->assertNotNull($address); + + // Simulate what a perealloc by engine code would leave behind: arData somewhere else + $table->getRawValue()->arData = Core::pointerAtAddress(Bucket::class, $address + 4096); + + $this->expectException(TypeOperationException::class); + $this->expectExceptionMessageMatches('/no longer the one in use/'); + $table->assertNoGrowth(); + } + + public function testAssertNoGrowthIsANoOpForEngineAllocatedStorage(): void + { + $table = new PersistentHashTable(); + $value = new ReflectionValue(1); + $table->add('grown by the engine', $value); + $value->release(); + + $table->assertNoGrowth(); + $this->assertFalse($table->hasExternalStorage()); + $this->assertSame(0, $table->getCapacity()); + $this->assertSame(0, $table->getRemainingCapacity()); + + $table->destroy(); + } + + public function testStructAndStorageCanBothLiveInTheSameForeignRegion(): void + { + // The end state the seam exists for: nothing of the table is in the process heap + $arena = new RecordingArenaAllocator(); + $capacity = 8; + $address = $arena->allocate( + PersistentHashTable::externalStorageSize($capacity), + Allocator::DEFAULT_ALIGNMENT, + ); + $table = PersistentHashTable::withExternalStorage($address, $capacity, $arena); + + $this->assertTrue($arena->holds($table->getRawValue())); + $this->assertTrue($arena->contains($address)); + + $value = new ReflectionValue('in the arena'); + $table->add('key', $value); + $value->release(); + + $this->assertSame('in the arena', self::valueOf($table->find('key'))); + + $arData = $table->getRawValue()->arData; + assert($arData !== null); + $this->assertTrue($arena->contains(Core::addressOf($arData)), 'The engine reallocated the buckets'); + $table->assertNoGrowth(); + } + + /** + * Reads the PHP value behind a lookup result, asserting the lookup found anything at all + */ + private static function valueOf(?ReflectionValue $value): mixed + { + self::assertNotNull($value); + $value->getNativeValue($native); + + return $native; + } + + /** + * Mints a persistent table with a freshly allocated external bucket block installed + */ + private function tableWithExternalStorage(int $capacity): PersistentHashTable + { + $address = EngineAllocator::persistent()->allocate( + PersistentHashTable::externalStorageSize($capacity), + Allocator::DEFAULT_ALIGNMENT, + ); + + return PersistentHashTable::withExternalStorage($address, $capacity); + } +} From fb35c54088ce434329c1a417030729c408110029 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 19:18:18 +0000 Subject: [PATCH 3/5] feat(core): name the object-store and class-table operations consumers need Packages built on z-engine had to reach through Core::$executor for two operations that have no public equivalent: registering an object the engine did not allocate (and handing its slot back), and probing the engine class table for a class name. Both are core-layer state per AGENTS.md, and the remedy for that is a named public method here rather than a reach-through there. ObjectStore::current() is the public entry point into EG(objects_store); register()/unregister() are the CData-free, ownership-explicit forms of put()/recycle() - the store never takes ownership, so the caller keeps the object memory and gives the slot back before releasing it. ReflectionClass::fromClassTable() answers "is this class already in the engine?" without autoloading and without throwing, which is what a re-attachment path needs before it trusts recorded class metadata. Refs #223 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B9xaBchjdo1atarNZ6sZqe --- src/Reflection/ReflectionClass.php | 23 ++++ src/System/ObjectStore.php | 44 ++++++++ tests/Reflection/ReflectionClassTest.php | 27 +++++ tests/System/ObjectStoreRegistrationTest.php | 106 +++++++++++++++++++ 4 files changed, 200 insertions(+) create mode 100644 tests/System/ObjectStoreRegistrationTest.php diff --git a/src/Reflection/ReflectionClass.php b/src/Reflection/ReflectionClass.php index 260763e..3f039d7 100644 --- a/src/Reflection/ReflectionClass.php +++ b/src/Reflection/ReflectionClass.php @@ -235,6 +235,29 @@ public static function fromCData(object $classEntry): ReflectionClass return $reflectionClass; } + /** + * Looks a class up in the engine class table WITHOUT autoloading it + * + * The named public form of the EG(class_table) lookup: the engine-global wrappers behind + * Core::$executor are core-layer state and not a consumer API (AGENTS.md), so a package + * asking "is this class already in the engine, and which entry is it?" asks here. + * + * Unlike the constructor (and unlike `new \ReflectionClass($name)`) this neither triggers + * the autoloader nor throws for a class the engine does not know: a miss is null, which is + * what makes it usable as a probe before re-attaching data recorded for that class name. + * + * Names are matched the way the engine keys its table, lowercased. + */ + public static function fromClassTable(string $className): ?ReflectionClass + { + $classEntryValue = Core::$executor->classTable->find(strtolower($className)); + if ($classEntryValue === null) { + return null; + } + + return static::fromCData($classEntryValue->getRawClass()); + } + /** * @inheritDoc */ diff --git a/src/System/ObjectStore.php b/src/System/ObjectStore.php index d01ed1f..894e555 100644 --- a/src/System/ObjectStore.php +++ b/src/System/ObjectStore.php @@ -109,6 +109,19 @@ public function offsetUnset($offset): void throw new \LogicException('Object store is read-only structure'); } + /** + * Returns the object store of the CURRENT request + * + * The public entry point into EG(objects_store): the engine-global wrappers behind + * Core::$executor are core-layer state and not a consumer API (AGENTS.md), so a package + * that has to hand the engine an object of its own asks here instead of reaching through + * them. The returned view is the one the executor itself uses - stores are never copies. + */ + public static function current(): self + { + return Core::$executor->objectStore; + } + /** * Returns the free head (aka next handle) */ @@ -117,6 +130,37 @@ public function nextHandle(): int return $this->pointer->free_list_head; } + /** + * Registers an object in this request's store and returns the handle it received + * + * The named, CData-free form of put(): an object that was NOT built by + * zend_object_std_init - a persistent clone re-attached for this request, say - becomes + * visible to the engine (spl_object_id, object iteration, var_dump) only after it is + * registered here, and every request needs its own registration because the store dies + * with the request. + * + * The store does not take ownership: the caller still owns the object memory and must + * unregister() the handle before that memory goes away, or the engine walks a bucket + * pointing at released memory at request shutdown. + */ + public function register(ObjectEntry $object): int + { + return $this->put($object->getRawValue()); + } + + /** + * Returns a handle obtained from register() to the store's free list + * + * The named, ownership-explicit form of recycle(): the bucket stops pointing at the + * object and the slot becomes reusable, without any destructor running and without the + * object itself being freed - which is exactly what an owner of externally allocated + * objects needs at the end of a request. + */ + public function unregister(int $handle): void + { + $this->recycle($handle); + } + /** * Registers an object in the store, assigning it a fresh handle * diff --git a/tests/Reflection/ReflectionClassTest.php b/tests/Reflection/ReflectionClassTest.php index 16b1164..5021708 100644 --- a/tests/Reflection/ReflectionClassTest.php +++ b/tests/Reflection/ReflectionClassTest.php @@ -766,6 +766,33 @@ public function testInstallExtensionHandlersWiresGetPropertiesFor(): void $this->assertContains('fields:' . TestPropertyHandlers::class, TestPropertyHandlers::$log); } + public function testFromClassTableFindsALoadedClassWithoutAutoloading(): void + { + $reflection = ReflectionClass::fromClassTable(TestClass::class); + + $this->assertNotNull($reflection); + $this->assertSame(TestClass::class, $reflection->getName()); + // The lookup is keyed the way the engine keys its table, so the case does not matter + $this->assertNotNull(ReflectionClass::fromClassTable(strtoupper(TestClass::class))); + } + + public function testFromClassTableReportsAnUnknownClassAsNullInsteadOfAutoloading(): void + { + $autoloaderCalls = []; + $autoloader = static function (string $className) use (&$autoloaderCalls): void { + $autoloaderCalls[] = $className; + }; + spl_autoload_register($autoloader); + try { + $missing = ReflectionClass::fromClassTable('ZEngine\\Stub\\ThereIsNoSuchClass'); + } finally { + spl_autoload_unregister($autoloader); + } + + $this->assertNull($missing); + $this->assertSame([], $autoloaderCalls, 'The class-table probe must not trigger autoloading'); + } + public function testInstallExtensionHandlers(): void { $refClass = new ReflectionClass(NativeNumber::class); diff --git a/tests/System/ObjectStoreRegistrationTest.php b/tests/System/ObjectStoreRegistrationTest.php new file mode 100644 index 0000000..af4b7b7 --- /dev/null +++ b/tests/System/ObjectStoreRegistrationTest.php @@ -0,0 +1,106 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\System; + +use PHPUnit\Framework\TestCase; +use ZEngine\Core; +use ZEngine\Reflection\ReflectionValue; +use ZEngine\Stub\TestPersistentCandidate; +use ZEngine\Type\ObjectEntry; +use ZEngine\Type\PersistentObjectFactory; + +/** + * The public object-store surface a package outside z-engine needs + * + * A package that mints objects the engine did not allocate (persistent clones re-attached + * for this request) has to register them for the request and hand the slot back afterwards. + * Both used to require reaching through Core::$executor, which is core-layer state and not + * a consumer API (AGENTS.md) - these are the named methods that replace the reach-through. + */ +class ObjectStoreRegistrationTest extends TestCase +{ + public function testCurrentReturnsTheStoreOfTheRunningRequest(): void + { + $store = ObjectStore::current(); + + $this->assertSame($store, ObjectStore::current()); + $this->assertGreaterThan(0, $store->count()); + + // It really is the store the engine uses: a live object is findable under its own id + $probe = new TestPersistentCandidate(); + $entry = $store[spl_object_id($probe)]; + $this->assertNotNull($entry); + $this->assertSame(TestPersistentCandidate::class, $entry->getClass()->getName()); + } + + public function testRegisterMakesAForeignObjectVisibleForThisRequest(): void + { + $source = new TestPersistentCandidate(); + $sourceValue = new ReflectionValue($source); + $clone = PersistentObjectFactory::persistentClone($sourceValue->getRawObject()); + $sourceValue->release(); + + $store = ObjectStore::current(); + $entry = ObjectEntry::fromCData($clone); + // A clone starts with the byte-copied handle of its source; registration is what + // gives it one of its own + $entry->setHandle(0); + + $handle = $store->register($entry); + $registeredEntry = $store[$handle]; + + $this->assertGreaterThan(0, $handle); + $this->assertSame($handle, $entry->getHandle()); + $this->assertTrue($store->offsetExists($handle)); + $this->assertNotNull($registeredEntry); + $this->assertSame(Core::addressOf($clone), Core::addressOf($registeredEntry->getRawValue())); + + // ... and unregister() gives the slot back without destroying anything + $store->unregister($handle); + $this->assertFalse($store->offsetExists($handle)); + + // The object itself survived the unregistration: its memory belongs to the caller + $this->assertSame(PersistentObjectFactory::PIN_BASELINE, $entry->getReferenceCount()); + + Core::untrackAndFree($clone); + } + + public function testACloneCanBeRegisteredAgainAfterBeingUnregistered(): void + { + $source = new TestPersistentCandidate(); + $sourceValue = new ReflectionValue($source); + $clone = PersistentObjectFactory::persistentClone($sourceValue->getRawObject()); + $sourceValue->release(); + + $store = ObjectStore::current(); + $entry = ObjectEntry::fromCData($clone); + $entry->setHandle(0); + + $first = $store->register($entry); + $store->unregister($first); + $second = $store->register($entry); + $registeredEntry = $store[$second]; + + // Every request re-registers the same clone, which is what makes a persistent object + // usable across requests: the handle is per-registration, the object is not + $this->assertGreaterThan(0, $second); + $this->assertSame($second, $entry->getHandle()); + $this->assertTrue($store->offsetExists($second)); + $this->assertNotNull($registeredEntry); + $this->assertSame(Core::addressOf($clone), Core::addressOf($registeredEntry->getRawValue())); + + $store->unregister($second); + Core::untrackAndFree($clone); + } +} From f8a4ab59ed85c8300f2834a3d4ed3024f097c39a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 19:48:50 +0000 Subject: [PATCH 4/5] refactor(system): encapsulate store registration on ObjectEntry per review ObjectStore::current() re-exposed Core::$executor->objectStore through a side door, and the register()/unregister() pair was only reachable through it - the trio recreated the very reach-through it was meant to remove, with an extra hop. The operations belong to the object being registered, not to a store handed out to callers. ObjectEntry::register() registers THIS object in the current request's store and returns the handle the engine assigned; unregister() reads the object's handle at call time and returns that slot to the free list. The slot is verified to actually hold this object first, so a stale handle - never registered, already unregistered, or a slot meanwhile reused - is refused with a named domain exception instead of silently detaching somebody else's object. ObjectStore is byte-for-byte what it was before this PR: put() and recycle() stay @internal, and a consumer never touches EG(objects_store) at all. Refs #223 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B9xaBchjdo1atarNZ6sZqe --- src/System/ObjectStore.php | 44 ------ src/Type/ObjectEntry.php | 46 ++++++ src/Type/TypeOperationException.php | 12 ++ tests/System/ObjectStoreRegistrationTest.php | 106 ------------- tests/Type/ObjectEntryRegistrationTest.php | 154 +++++++++++++++++++ 5 files changed, 212 insertions(+), 150 deletions(-) delete mode 100644 tests/System/ObjectStoreRegistrationTest.php create mode 100644 tests/Type/ObjectEntryRegistrationTest.php diff --git a/src/System/ObjectStore.php b/src/System/ObjectStore.php index 894e555..d01ed1f 100644 --- a/src/System/ObjectStore.php +++ b/src/System/ObjectStore.php @@ -109,19 +109,6 @@ public function offsetUnset($offset): void throw new \LogicException('Object store is read-only structure'); } - /** - * Returns the object store of the CURRENT request - * - * The public entry point into EG(objects_store): the engine-global wrappers behind - * Core::$executor are core-layer state and not a consumer API (AGENTS.md), so a package - * that has to hand the engine an object of its own asks here instead of reaching through - * them. The returned view is the one the executor itself uses - stores are never copies. - */ - public static function current(): self - { - return Core::$executor->objectStore; - } - /** * Returns the free head (aka next handle) */ @@ -130,37 +117,6 @@ public function nextHandle(): int return $this->pointer->free_list_head; } - /** - * Registers an object in this request's store and returns the handle it received - * - * The named, CData-free form of put(): an object that was NOT built by - * zend_object_std_init - a persistent clone re-attached for this request, say - becomes - * visible to the engine (spl_object_id, object iteration, var_dump) only after it is - * registered here, and every request needs its own registration because the store dies - * with the request. - * - * The store does not take ownership: the caller still owns the object memory and must - * unregister() the handle before that memory goes away, or the engine walks a bucket - * pointing at released memory at request shutdown. - */ - public function register(ObjectEntry $object): int - { - return $this->put($object->getRawValue()); - } - - /** - * Returns a handle obtained from register() to the store's free list - * - * The named, ownership-explicit form of recycle(): the bucket stops pointing at the - * object and the slot becomes reusable, without any destructor running and without the - * object itself being freed - which is exactly what an owner of externally allocated - * objects needs at the end of a request. - */ - public function unregister(int $handle): void - { - $this->recycle($handle); - } - /** * Registers an object in the store, assigning it a fresh handle * diff --git a/src/Type/ObjectEntry.php b/src/Type/ObjectEntry.php index 5b6312e..befde3c 100644 --- a/src/Type/ObjectEntry.php +++ b/src/Type/ObjectEntry.php @@ -169,6 +169,52 @@ public function setHandle(int $newHandle): void $this->pointer->handle = $newHandle; } + /** + * Registers this object in the object store of the CURRENT request + * + * An object the engine did not allocate through zend_object_std_init - a persistent + * clone re-attached for this request, an object minted into foreign memory - is + * invisible to the engine until it holds a slot in EG(objects_store): spl_object_id(), + * object iteration and every store walk go through that table. The store dies with the + * request, so a cross-request object is registered again in every request that uses it. + * + * The store does NOT take ownership: the object memory stays with whoever allocated it, + * and it must be unregister()ed before that memory goes away - a bucket still pointing + * at released memory is walked by the engine at request shutdown. + * + * @return int The handle the engine assigned (== spl_object_id of this object) + */ + public function register(): int + { + $this->assertObjectAlive(); + + return Core::$executor->objectStore->put($this->pointer); + } + + /** + * Returns this object's store slot to the free list, leaving the object itself untouched + * + * The counterpart of register(): the bucket stops pointing at this object and the slot + * becomes reusable, without any destructor running and without the object being freed. + * The handle is read from the object at call time, and the slot is verified to actually + * hold THIS object before it is recycled - a stale handle (never registered, already + * unregistered, or a slot meanwhile reused by another object) is refused instead of + * silently detaching somebody else's object. + */ + public function unregister(): void + { + $this->assertObjectAlive(); + + $store = Core::$executor->objectStore; + $handle = $this->pointer->handle; + $bucket = $store->offsetExists($handle) ? $store[$handle] : null; + if ($bucket === null || Core::addressOf($bucket->getRawValue()) !== Core::addressOf($this->pointer)) { + throw TypeOperationException::objectNotRegistered($handle); + } + + $store->recycle($handle); + } + /** * Checks if this object is a lazy object (PHP 8.4 ReflectionClass::newLazyGhost()/newLazyProxy()) * diff --git a/src/Type/TypeOperationException.php b/src/Type/TypeOperationException.php index f5470b0..e398250 100644 --- a/src/Type/TypeOperationException.php +++ b/src/Type/TypeOperationException.php @@ -135,6 +135,18 @@ public static function storageRelocated(): self ); } + /** + * Raised when an object is unregistered from a store slot that does not hold it + */ + public static function objectNotRegistered(int $handle): self + { + return new self( + "This object does not occupy the object store slot {$handle}: it was never " + . 'registered in this request, has already been unregistered, or the slot has ' + . 'meanwhile been reused by another object', + ); + } + public static function danglingObjectEntry(): self { return new self('The underlying object has been destroyed, this entry is dangling'); diff --git a/tests/System/ObjectStoreRegistrationTest.php b/tests/System/ObjectStoreRegistrationTest.php deleted file mode 100644 index af4b7b7..0000000 --- a/tests/System/ObjectStoreRegistrationTest.php +++ /dev/null @@ -1,106 +0,0 @@ - - * - * This source file is subject to the license that is bundled - * with this source code in the file LICENSE. - * - */ -declare(strict_types=1); - -namespace ZEngine\System; - -use PHPUnit\Framework\TestCase; -use ZEngine\Core; -use ZEngine\Reflection\ReflectionValue; -use ZEngine\Stub\TestPersistentCandidate; -use ZEngine\Type\ObjectEntry; -use ZEngine\Type\PersistentObjectFactory; - -/** - * The public object-store surface a package outside z-engine needs - * - * A package that mints objects the engine did not allocate (persistent clones re-attached - * for this request) has to register them for the request and hand the slot back afterwards. - * Both used to require reaching through Core::$executor, which is core-layer state and not - * a consumer API (AGENTS.md) - these are the named methods that replace the reach-through. - */ -class ObjectStoreRegistrationTest extends TestCase -{ - public function testCurrentReturnsTheStoreOfTheRunningRequest(): void - { - $store = ObjectStore::current(); - - $this->assertSame($store, ObjectStore::current()); - $this->assertGreaterThan(0, $store->count()); - - // It really is the store the engine uses: a live object is findable under its own id - $probe = new TestPersistentCandidate(); - $entry = $store[spl_object_id($probe)]; - $this->assertNotNull($entry); - $this->assertSame(TestPersistentCandidate::class, $entry->getClass()->getName()); - } - - public function testRegisterMakesAForeignObjectVisibleForThisRequest(): void - { - $source = new TestPersistentCandidate(); - $sourceValue = new ReflectionValue($source); - $clone = PersistentObjectFactory::persistentClone($sourceValue->getRawObject()); - $sourceValue->release(); - - $store = ObjectStore::current(); - $entry = ObjectEntry::fromCData($clone); - // A clone starts with the byte-copied handle of its source; registration is what - // gives it one of its own - $entry->setHandle(0); - - $handle = $store->register($entry); - $registeredEntry = $store[$handle]; - - $this->assertGreaterThan(0, $handle); - $this->assertSame($handle, $entry->getHandle()); - $this->assertTrue($store->offsetExists($handle)); - $this->assertNotNull($registeredEntry); - $this->assertSame(Core::addressOf($clone), Core::addressOf($registeredEntry->getRawValue())); - - // ... and unregister() gives the slot back without destroying anything - $store->unregister($handle); - $this->assertFalse($store->offsetExists($handle)); - - // The object itself survived the unregistration: its memory belongs to the caller - $this->assertSame(PersistentObjectFactory::PIN_BASELINE, $entry->getReferenceCount()); - - Core::untrackAndFree($clone); - } - - public function testACloneCanBeRegisteredAgainAfterBeingUnregistered(): void - { - $source = new TestPersistentCandidate(); - $sourceValue = new ReflectionValue($source); - $clone = PersistentObjectFactory::persistentClone($sourceValue->getRawObject()); - $sourceValue->release(); - - $store = ObjectStore::current(); - $entry = ObjectEntry::fromCData($clone); - $entry->setHandle(0); - - $first = $store->register($entry); - $store->unregister($first); - $second = $store->register($entry); - $registeredEntry = $store[$second]; - - // Every request re-registers the same clone, which is what makes a persistent object - // usable across requests: the handle is per-registration, the object is not - $this->assertGreaterThan(0, $second); - $this->assertSame($second, $entry->getHandle()); - $this->assertTrue($store->offsetExists($second)); - $this->assertNotNull($registeredEntry); - $this->assertSame(Core::addressOf($clone), Core::addressOf($registeredEntry->getRawValue())); - - $store->unregister($second); - Core::untrackAndFree($clone); - } -} diff --git a/tests/Type/ObjectEntryRegistrationTest.php b/tests/Type/ObjectEntryRegistrationTest.php new file mode 100644 index 0000000..6d28fb5 --- /dev/null +++ b/tests/Type/ObjectEntryRegistrationTest.php @@ -0,0 +1,154 @@ + + * + * This source file is subject to the license that is bundled + * with this source code in the file LICENSE. + * + */ +declare(strict_types=1); + +namespace ZEngine\Type; + +use PHPUnit\Framework\TestCase; +use ZEngine\Core; +use ZEngine\Generated\zend_object; +use ZEngine\Reflection\ReflectionValue; +use ZEngine\Stub\TestPersistentCandidate; + +/** + * Store registration seen from the object that is being registered + * + * A package that mints objects the engine did not allocate (persistent clones re-attached + * for this request) has to make them visible for the request and hand the slot back + * afterwards. Both operations belong to the object, not to a store handed out to callers: + * the engine-global wrappers behind Core::$executor stay core-layer state (AGENTS.md) and + * a consumer never touches EG(objects_store) at all. + */ +class ObjectEntryRegistrationTest extends TestCase +{ + public function testRegisterMakesAForeignObjectVisibleForThisRequest(): void + { + $clone = $this->persistentClone(); + $entry = ObjectEntry::fromCData($clone); + // A clone starts with the byte-copied handle of its source; registration is what + // gives it one of its own + $entry->setHandle(0); + + $handle = $entry->register(); + + $this->assertGreaterThan(0, $handle); + $this->assertSame($handle, $entry->getHandle()); + + // It really is in the store the engine uses: the object materializes under its + // own handle, and spl_object_id() agrees with it + $instance = $entry->getNativeValue(); + $this->assertSame($handle, spl_object_id($instance)); + $this->assertSame(TestPersistentCandidate::class, $instance::class); + + $entry->unregister(); + Core::untrackAndFree($clone); + } + + public function testUnregisterReturnsTheSlotWithoutTouchingTheObject(): void + { + $clone = $this->persistentClone(); + $entry = ObjectEntry::fromCData($clone); + $entry->setHandle(0); + + $handle = $entry->register(); + $entry->unregister(); + + // The slot is free again ... + $this->assertFalse(Core::$executor->objectStore->offsetExists($handle)); + // ... and the object itself survived: its memory belongs to the caller, and the + // pinned refcount of a persistent clone is untouched by the detach + $this->assertSame(PersistentObjectFactory::PIN_BASELINE, $entry->getReferenceCount()); + $this->assertTrue($entry->isPersistent()); + + Core::untrackAndFree($clone); + } + + public function testACloneCanBeRegisteredAgainAfterBeingUnregistered(): void + { + $clone = $this->persistentClone(); + $entry = ObjectEntry::fromCData($clone); + $entry->setHandle(0); + + $entry->register(); + $entry->unregister(); + $second = $entry->register(); + + // Every request re-registers the same clone, which is what makes a persistent object + // usable across requests: the handle is per-registration, the object is not + $this->assertGreaterThan(0, $second); + $this->assertSame($second, $entry->getHandle()); + $this->assertSame($second, spl_object_id($entry->getNativeValue())); + + $entry->unregister(); + Core::untrackAndFree($clone); + } + + public function testUnregisteringATwiceDetachedObjectIsRefused(): void + { + $clone = $this->persistentClone(); + $entry = ObjectEntry::fromCData($clone); + $entry->setHandle(0); + + $entry->register(); + $entry->unregister(); + + // The stale handle now points at a free (or meanwhile reused) slot: recycling it + // again would hand somebody else's object to the free list + $this->expectException(TypeOperationException::class); + $this->expectExceptionMessageMatches('/does not occupy the object store slot/'); + try { + $entry->unregister(); + } finally { + Core::untrackAndFree($clone); + } + } + + public function testUnregisteringANeverRegisteredObjectIsRefused(): void + { + $clone = $this->persistentClone(); + $entry = ObjectEntry::fromCData($clone); + $entry->setHandle(0); + + $this->expectException(TypeOperationException::class); + $this->expectExceptionMessageMatches('/never registered in this request/'); + try { + $entry->unregister(); + } finally { + Core::untrackAndFree($clone); + } + } + + public function testAnObjectTheEngineOwnsReportsItsOwnStoreSlot(): void + { + // An ordinary object is already registered by zend_object_std_init: the handle the + // wrapper reports is the very slot spl_object_id() names + $instance = new TestPersistentCandidate(); + $entry = ObjectEntry::weakFor($instance); + + $this->assertSame(spl_object_id($instance), $entry->getHandle()); + $this->assertTrue(Core::$executor->objectStore->offsetExists($entry->getHandle())); + } + + /** + * Mints a persistent clone of a fresh scalar-only object + * + * @return zend_object zend_object* of the clone, owned by the caller + */ + private function persistentClone(): object + { + $sourceValue = new ReflectionValue(new TestPersistentCandidate()); + $clone = PersistentObjectFactory::persistentClone($sourceValue->getRawObject()); + $sourceValue->release(); + + return $clone; + } +} From 1a6acb69c6e012e32641a97b2fc88a46a56906c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 19:57:37 +0000 Subject: [PATCH 5/5] fix(type): drop the materialized alias before freeing a registered clone The registration test released the clone's memory while a value materialized from it was still in scope. That value is a real PHP alias of the very same zend_object, so destroying it at the end of the test method made the engine write a refcount into a block that had already been handed back to malloc - and gc.refcount sits at offset 0 of a zend_object, which is exactly where glibc keeps the tcache next pointer of a free chunk. The next persistent clone of the same size popped that chunk and the allocator aborted, far away from the mistake: malloc(): unaligned tcache chunk detected (SIGABRT, exit 134) Repro (deterministic, aborts during the SECOND test of the class - the one whose clone lands in the poisoned size class): MALLOC_CHECK_=3 MALLOC_PERTURB_=85 php8.4 -d ffi.enable=1 \ -d opcache.jit=off vendor/bin/phpunit --filter ObjectEntryRegistration The registration path itself is sound: unregister() bounds-checks the handle against the store top and validates the bucket through the store's own tagged-pointer check before it compares identity, so it never dereferences a free-list sentinel. What was missing is the other half of the ownership contract, now spelled out on register(): registration makes the object reachable from userland, and every alias materialized from it must be gone before the memory is released, not just the store slot. Refs #223 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B9xaBchjdo1atarNZ6sZqe --- src/Type/ObjectEntry.php | 8 ++++++++ tests/Type/ObjectEntryRegistrationTest.php | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Type/ObjectEntry.php b/src/Type/ObjectEntry.php index befde3c..8658826 100644 --- a/src/Type/ObjectEntry.php +++ b/src/Type/ObjectEntry.php @@ -182,6 +182,14 @@ public function setHandle(int $newHandle): void * and it must be unregister()ed before that memory goes away - a bucket still pointing * at released memory is walked by the engine at request shutdown. * + * Registration makes the object REACHABLE FROM USERLAND, and that outlives the store + * slot: every value materialized from it (getNativeValue(), a lookup through the store, + * anything var_dump() handed out) is a real PHP alias whose release writes a refcount + * into the object. Such aliases must therefore be gone BEFORE the memory is released + * too - the write lands at offset 0 of the block, which is exactly where an allocator + * keeps its free-list bookkeeping, so a late alias corrupts the heap instead of + * segfaulting where the mistake was made. + * * @return int The handle the engine assigned (== spl_object_id of this object) */ public function register(): int diff --git a/tests/Type/ObjectEntryRegistrationTest.php b/tests/Type/ObjectEntryRegistrationTest.php index 6d28fb5..84735ee 100644 --- a/tests/Type/ObjectEntryRegistrationTest.php +++ b/tests/Type/ObjectEntryRegistrationTest.php @@ -48,6 +48,10 @@ public function testRegisterMakesAForeignObjectVisibleForThisRequest(): void $instance = $entry->getNativeValue(); $this->assertSame($handle, spl_object_id($instance)); $this->assertSame(TestPersistentCandidate::class, $instance::class); + // The materialized value is a REAL PHP alias of the clone: releasing it makes the + // engine write a refcount into the object, so it has to be gone before the memory + // is - the same ordering rule the unregister() contract states for the store slot + unset($instance); $entry->unregister(); Core::untrackAndFree($clone); @@ -84,9 +88,12 @@ public function testACloneCanBeRegisteredAgainAfterBeingUnregistered(): void // Every request re-registers the same clone, which is what makes a persistent object // usable across requests: the handle is per-registration, the object is not + $instance = $entry->getNativeValue(); $this->assertGreaterThan(0, $second); $this->assertSame($second, $entry->getHandle()); - $this->assertSame($second, spl_object_id($entry->getNativeValue())); + $this->assertSame($second, spl_object_id($instance)); + // Aliases go before the memory does (see the first test) + unset($instance); $entry->unregister(); Core::untrackAndFree($clone);