Item cache: hook response handler instead of racing the engine

`Item:CreateFromItemID(X):ContinueOnItemLoad(cb)` was leaving items
permanently uncached when pfQuest called it at addon-load /
PLAYER_ENTERING_WORLD against an empty WDB. Three intertwined bugs:

1. `GetItemInfoInstant` returned no values for uncached items, so
   `ItemMixin:GetItemID()` returned nil and `AddCallback(nil, cb)`
   crashed with "table index is nil".
2. `DoesItemExistByID` returned false for uncached items, which made
   `IsItemEmpty()` return true and erroneously gated `ContinueOnItemLoad`
   on cache state — modern semantic is "is this a valid ID", independent
   of cache.
3. Our `RequestLoadItemData` path eagerly called `CacheFetch` with our
   own callback at addon-load. That created a cache entry before the
   engine's natural inventory prefetch had a chance to run; the engine
   then saw the entry already existed and skipped its own SMSG, leaving
   the entry permanently pending. `GetItemInfo` retries didn't help —
   subsequent `_GetRecord` calls just appended callbacks to the dead
   list, no new query was sent.

Fixes:

- `GetItemInfoInstant` always pushes the itemID (echoed from input) and
  the remaining 6 fields when cached / nils when not. No cache touch.
- `DoesItemExistByID` returns `itemID > 0`. No cache touch.
- `RequestLoadItemData` tracks itemIDs in a pending set without touching
  the cache. We hook `FUN_ITEMSTATS_CACHE_RESPONSE` (the engine's
  `SMSG_ITEM_QUERY_SINGLE_RESPONSE` handler at `0x0055BDB0`) and sweep
  pending after the engine fills entries — observing the natural
  prefetch instead of racing it. `WorldTick` only handles escalation
  (after ~60 ticks, kick a query for items the engine doesn't prefetch
  itself, e.g. quest rewards) and the hard timeout.

Also fix the truthiness bug in modern (id, success) events:
`Event::Custom::Fire("%d%d", id, success)` was pushing `success` as a
Lua number — both 0 and 1 are truthy, so `if success then` fired the
success branch on failed loads. New `Event::Custom::FireIdSuccess`
pushes `1` for success and `nil` for failure (via the engine's
`lua_pushstring(NULL) → lua_pushnil` tail-jump). `ITEM_DATA_LOAD_RESULT`,
`GET_ITEM_INFO_RECEIVED`, and `QUEST_DATA_LOAD_RESULT` now all use this.

Author: brues-code

src/Offsets.h

@@ -1068,6 +1068,26 @@ enum Offsets {
     // With `requestIfMissing=false`, returns NULL if not in cache (no server
     // round-trip) — exactly the "instant" semantics we want.
     FUN_DBCACHE_ITEMSTATS_GET_RECORD = 0x0055BA30,
+    // Item-cache response handler — the function called from
+    // `0x00555140` (success path for `SMSG_ITEM_QUERY_SINGLE_RESPONSE`)
+    // and `0x00555160` (failure path). Parses one or more itemIDs out
+    // of the packet, looks up (or creates) the cache entry per ID,
+    // copies the item data block via `FUN_007c9640` to `entry+0x18`,
+    // sets `[entry+0x1F0]=1` (the loaded byte), then walks the entry's
+    // pending callback list at `[entry+0x1FC]` and invokes each with
+    // `(userData, success)`. High-bit-on itemID in the packet signals
+    // a per-item failure for the batch.
+    //
+    // We hook this to observe any item-cache fill the engine performs
+    // — covers the engine's natural inventory prefetch and any
+    // addon-triggered queries without having to register our own
+    // engine callbacks. Lets `RequestLoadItemData` be purely passive:
+    // we track itemIDs, the engine (or whoever) issues the query, we
+    // see the fill via this hook and fire `ITEM_DATA_LOAD_RESULT`.
+    //
+    // Signature: `__thiscall(void *cache, void *packetReader, int flag)`.
+    // `cache` is the same `VAR_ITEMDB_CACHE` we already use.
+    FUN_ITEMSTATS_CACHE_RESPONSE = 0x0055BDB0,
     // ItemStats_C field offsets we read. Full struct layout in
     // VanillaHelpers's `Game.h` (`struct ItemStats_C`); we only need these.
     OFF_ITEMSTATS_CLASS = 0x00,

src/event/Custom.h

@@ -74,6 +74,22 @@ inline void Fire(int eventID, const char *format, Args... args) {
     fn(eventID, format, args...);
 }
 
+// Convenience for the modern WoW `(id: number, success: bool)` event
+// shape (ITEM_DATA_LOAD_RESULT, GET_ITEM_INFO_RECEIVED,
+// QUEST_DATA_LOAD_RESULT, …). The engine's printf-style dispatcher has
+// no `%b`, so a naive `%d%d` with `0` for failure surfaces `arg2 = 0` —
+// truthy in Lua, breaking the canonical `if success then` branch. We
+// push `1` for success and `nil` for failure instead, leaning on the
+// engine's `lua_pushstring(L, NULL) → lua_pushnil` tail-jump so the
+// dispatcher itself emits the nil without any pre-staging. Lua handlers
+// can then use the natural idiom: nil is falsy, `1` is truthy.
+inline void FireIdSuccess(int eventID, int id, bool success) {
+    if (success)
+        Fire(eventID, "%d%d", id, 1);
+    else
+        Fire(eventID, "%d%s", id, static_cast<const char *>(nullptr));
+}
+
 // Internal: try to claim a slot for every reservation that's still
 // unclaimed (`slot < 0`). Called from the `Frame::RegisterEvent` hook
 // in DllMain — every time Lua calls `frame:RegisterEvent(...)`, we

src/item/Data.cpp

@@ -17,6 +17,7 @@
 #include "item/Arg.h"
 #include "item/Data.h"
 #include "item/Location.h"
+#include "tick/WorldTick.h"
 
 #include <cstdint>
 
@@ -52,16 +53,14 @@ static constexpr const char *kGetItemInfoReceived = "GET_ITEM_INFO_RECEIVED";
 static const Event::Custom::AutoReserve _reserveItemDataLoadResult{kItemDataLoadResult};
 static const Event::Custom::AutoReserve _reserveGetItemInfoReceived{kGetItemInfoReceived};
 
-static void FireGetItemInfoReceived(int itemID, int success) {
-    const int slot = Event::Custom::Lookup(kGetItemInfoReceived);
-    if (slot >= 0)
-        Event::Custom::Fire(slot, "%d%d", itemID, success);
+static void FireGetItemInfoReceived(int itemID, bool success) {
+    Event::Custom::FireIdSuccess(Event::Custom::Lookup(kGetItemInfoReceived),
+                                 itemID, success);
 }
 
-static void FireItemDataLoadResult(int itemID, int success) {
-    const int slot = Event::Custom::Lookup(kItemDataLoadResult);
-    if (slot >= 0)
-        Event::Custom::Fire(slot, "%d%d", itemID, success);
+static void FireItemDataLoadResult(int itemID, bool success) {
+    Event::Custom::FireIdSuccess(Event::Custom::Lookup(kItemDataLoadResult),
+                                 itemID, success);
 }
 
 // Engine callback for **implicit** cache fills (transparent warmup
@@ -76,16 +75,7 @@ static void FireItemDataLoadResult(int itemID, int success) {
 // callback would crash the engine, so stdcall is verified by induction.
 static void __stdcall ItemLoadCallback_Implicit(void *userData, int success) {
     const auto itemID = static_cast<int>(reinterpret_cast<uintptr_t>(userData));
-    FireGetItemInfoReceived(itemID, static_cast<int>(success != 0));
-}
-
-// Engine callback for **explicit** cache fills (player-side
-// `C_Item.RequestLoadItemData(ByID)` calls). Fires ITEM_DATA_LOAD_RESULT
-// only — the implicit GET_ITEM_INFO_RECEIVED path is for ambient cache
-// fills, not for callers who explicitly asked.
-static void __stdcall ItemLoadCallback_Explicit(void *userData, int success) {
-    const auto itemID = static_cast<int>(reinterpret_cast<uintptr_t>(userData));
-    FireItemDataLoadResult(itemID, static_cast<int>(success != 0));
+    FireGetItemInfoReceived(itemID, success != 0);
 }
 
 using ItemLoadCallback_t = void(__stdcall *)(void *userData, int success);
@@ -148,18 +138,143 @@ static int __fastcall Script_IsItemDataCached(void *L) {
     return 1;
 }
 
-// Explicit-request path: kicks off the cache fill via the explicit
-// callback (which fires ITEM_DATA_LOAD_RESULT). If the item is already
-// cached, the engine won't invoke our callback — synthesize the event
-// so addons get the same notification regardless of cache state,
-// matching modern `C_Item.RequestLoadItemData(ByID)` semantics.
+// Pending explicit-request tracking. We deliberately do NOT eagerly
+// create cache entries when an addon calls `C_Item.RequestLoadItemData`
+// at addon-load / PLAYER_ENTERING_WORLD time — empirically (pfQuest),
+// pre-creating the entry blocks the engine's natural prefetch path
+// from issuing a fresh `SMSG_ITEM_QUERY_SINGLE`. The entry stays in a
+// permanently-pending state because nothing ever sends the query.
+//
+// Instead we hook the engine's item-cache response handler at
+// `FUN_ITEMSTATS_CACHE_RESPONSE` (the function that fills cache
+// entries on `SMSG_ITEM_QUERY_SINGLE_RESPONSE`). After the engine
+// finishes filling any entries and walking its own callback lists,
+// we sweep our pending set and fire `ITEM_DATA_LOAD_RESULT(itemID, 1)`
+// for any tracked items that now resolve. The engine's natural
+// prefetch (for player-inventory items) takes care of the actual
+// `SMSG_ITEM_QUERY_SINGLE` for us.
+//
+// For items the engine *doesn't* naturally prefetch (e.g. quest
+// reward IDs the player has never owned), we escalate after
+// `kEscalateTicks` by issuing our own `CacheFetch` with a no-op
+// callback — that creates the entry and queues the query, and the
+// response handler hook still fires our event when the data lands.
+// `kMaxWaitTicks` is the hard timeout that gives up and fires
+// `ITEM_DATA_LOAD_RESULT(itemID, nil)`.
+struct Pending {
+    uint32_t itemID;
+    int ticksWaiting;
+    bool requestIssued;
+};
+
+constexpr int kMaxPending = 64;
+constexpr int kEscalateTicks = 60;
+constexpr int kMaxWaitTicks = 1200;
+
+static Pending g_pending[kMaxPending];
+static int g_pendingCount = 0;
+
+static int FindPending(uint32_t itemID) {
+    for (int i = 0; i < g_pendingCount; ++i) {
+        if (g_pending[i].itemID == itemID)
+            return i;
+    }
+    return -1;
+}
+
+static void RemovePendingAt(int idx) {
+    --g_pendingCount;
+    if (idx != g_pendingCount)
+        g_pending[idx] = g_pending[g_pendingCount];
+}
+
+// No-op callback for the escalation path — we just need a non-null
+// callback pointer so the engine's `_GetRecord` queues the query.
+// The response handler hook will detect the fill and fire the event;
+// this callback doesn't need to do anything.
+static void __stdcall ItemLoadCallback_NoOp(void * /*userData*/, int /*success*/) {}
+
 static void RequestAndMaybeNotify(uint32_t itemID) {
-    const bool wasCached = (CacheFetch(itemID, nullptr) != nullptr);
-    CacheFetch(itemID, &ItemLoadCallback_Explicit);
-    if (wasCached)
-        FireItemDataLoadResult(static_cast<int>(itemID), 1);
+    // Already loaded — fire success synchronously.
+    if (CacheFetch(itemID, nullptr) != nullptr) {
+        FireItemDataLoadResult(static_cast<int>(itemID), true);
+        return;
+    }
+    // Already tracking — second request just dedups.
+    if (FindPending(itemID) >= 0)
+        return;
+    if (g_pendingCount >= kMaxPending)
+        return;
+    g_pending[g_pendingCount].itemID = itemID;
+    g_pending[g_pendingCount].ticksWaiting = 0;
+    g_pending[g_pendingCount].requestIssued = false;
+    ++g_pendingCount;
+}
+
+// Response handler hook. After the engine fills cache entries from
+// `SMSG_ITEM_QUERY_SINGLE_RESPONSE` and walks the per-entry callback
+// lists, we sweep `g_pending` for anything now-loaded and fire the
+// event. This is the engine-aligned alternative to per-tick polling:
+// we observe fills exactly when they happen, with zero added latency
+// and no risk of racing the engine's own callback walk.
+//
+// `__thiscall` is mimicked as `__fastcall(ecx, edx_unused, ...)` per
+// the convention used elsewhere in this codebase (see
+// `KeyringDescChange_h`, `FrameRegisterEvent_h`).
+using ItemStatsCacheResponse_t = void(__thiscall *)(void *cache, void *packetReader, int flag);
+static ItemStatsCacheResponse_t ItemStatsCacheResponse_o = nullptr;
+
+static void __fastcall ItemStatsCacheResponse_h(void *cache, void * /*edx_unused*/,
+                                                 void *packetReader, int flag) {
+    using Trampoline_t = void(__fastcall *)(void *cache, void *edx, void *packetReader, int flag);
+    reinterpret_cast<Trampoline_t>(ItemStatsCacheResponse_o)(cache, nullptr, packetReader, flag);
+    // Sweep pending — the engine just finished filling any matching
+    // entries. Peek (no callback) returns non-null iff the entry's
+    // loaded flag is set.
+    for (int i = 0; i < g_pendingCount;) {
+        const uint32_t itemID = g_pending[i].itemID;
+        if (CacheFetch(itemID, nullptr) != nullptr) {
+            RemovePendingAt(i);
+            FireItemDataLoadResult(static_cast<int>(itemID), true);
+            continue;
+        }
+        ++i;
+    }
+}
+
+static const Game::HookAutoRegister _hookCacheResponse{
+    Offsets::FUN_ITEMSTATS_CACHE_RESPONSE,
+    reinterpret_cast<void *>(&ItemStatsCacheResponse_h),
+    reinterpret_cast<void **>(&ItemStatsCacheResponse_o)};
+
+// Escalation + timeout. The hook covers the happy path (engine fills
+// cache → we fire). This tick is only responsible for:
+//   1. Issuing a query for items the engine doesn't naturally prefetch
+//      (e.g. quest rewards) — wait `kEscalateTicks` so we don't race
+//      the engine's startup-time state machine, then call CacheFetch
+//      with a no-op callback. The engine sends the SMSG, the response
+//      handler hook fires the event when the response lands.
+//   2. Giving up after `kMaxWaitTicks` for items that never resolve
+//      (bad itemID, server doesn't respond, etc.) — fire failure.
+static void OnWorldTick() {
+    for (int i = 0; i < g_pendingCount;) {
+        if (!g_pending[i].requestIssued && g_pending[i].ticksWaiting >= kEscalateTicks) {
+            CacheFetch(g_pending[i].itemID, &ItemLoadCallback_NoOp);
+            g_pending[i].requestIssued = true;
+        }
+        if (g_pending[i].ticksWaiting >= kMaxWaitTicks) {
+            const uint32_t itemID = g_pending[i].itemID;
+            RemovePendingAt(i);
+            FireItemDataLoadResult(static_cast<int>(itemID), false);
+            continue;
+        }
+        ++g_pending[i].ticksWaiting;
+        ++i;
+    }
 }
 
+static const Tick::WorldTick::AutoSubscribe _tickSub{&OnWorldTick};
+
 // Public C++ API — see Item/Data.h. Implicit (transparent) warmup
 // path used by the `Script_GetItemInfo` hook and `SetItemByID`. Fires
 // the cache request only if the item isn't already cached; uses the

src/item/Exists.cpp

@@ -14,28 +14,12 @@
 #include "Game.h"
 #include "Offsets.h"
 #include "item/Arg.h"
-#include "item/Data.h"
 #include "item/Location.h"
 
 #include <cstdint>
 
 namespace Item::Exists {
 
-namespace {
-
-using GetItemRecord_t = const uint8_t *(__thiscall *)(void *cache, uint32_t itemID,
-                                                      const uint64_t *guid, void *callback,
-                                                      void *userData, int unused);
-
-const uint8_t *PeekItemRecord(uint32_t itemID) {
-    auto fn = reinterpret_cast<GetItemRecord_t>(Offsets::FUN_DBCACHE_ITEMSTATS_GET_RECORD);
-    auto *cache = reinterpret_cast<void *>(Offsets::VAR_ITEMDB_CACHE);
-    const uint64_t zeroGuid = 0;
-    return fn(cache, itemID, &zeroGuid, nullptr, nullptr, 0);
-}
-
-} // namespace
-
 // `C_Item.DoesItemExist(itemLocation)` — true iff the location resolves
 // to a populated inventory slot on the active player. Equipment-slot
 // locations check the character pane; bag-slot locations walk the
@@ -51,27 +35,32 @@ static int __fastcall Script_C_Item_DoesItemExist(void *L) {
     return 1;
 }
 
-// `C_Item.DoesItemExistByID(itemInfo)` — true iff the cache currently
-// has data for this itemID (i.e. `GetItemInfo` would return non-nil
-// right now). Same semantics as the modern function and the existing
-// Lua shim: cache miss → false, but we kick off the network query
-// so a subsequent call after `GET_ITEM_INFO_RECEIVED` will succeed.
+// `C_Item.DoesItemExistByID(itemInfo)` — true if `itemID` looks like a
+// valid item identifier. Modern WoW answers from the client-side item
+// database (Item-sparse.db2) and is independent of whether the network
+// cache has been warmed yet. 1.12 has no client-side DB, so we
+// optimistically accept any positive itemID; callers needing the
+// "is the cache populated?" check should use `IsItemDataCachedByID`.
+//
+// This split matters for `ItemMixin:IsItemEmpty` (FrameXML ItemUtil),
+// which uses `DoesItemExistByID` to gate `ContinueOnItemLoad` — the
+// whole point of `ContinueOnItemLoad` is to wait for an uncached item
+// to load, so the existence check must succeed for items that aren't
+// in the cache yet.
+//
+// We deliberately do NOT auto-warm the cache here. Existence is a pure
+// query about the ID, not a load request — and during early-login
+// (cold WDB, pfQuest's PLAYER_ENTERING_WORLD path) a spurious cache
+// touch can race the engine's natural inventory prefetch and leave the
+// item stuck. Callers wanting to actually load data should use
+// `RequestLoadItemDataByID`; passive readers (`GetItemInfo`,
+// `GetItemNameByID`) already warm on their own miss path.
 //
 // Accepts numeric itemID or `"item:NNN..."` string (including full
-// chat links). Returns false for any input we can't resolve to an
-// itemID; never raises.
+// chat links); never raises.
 static int __fastcall Script_C_Item_DoesItemExistByID(void *L) {
     const int itemID = Item::Arg::ResolveItemID(L, 1);
-    if (itemID <= 0) {
-        Game::Lua::PushBoolean(L, 0);
-        return 1;
-    }
-    if (PeekItemRecord(static_cast<uint32_t>(itemID)) == nullptr) {
-        Item::Data::WarmCache(static_cast<uint32_t>(itemID));
-        Game::Lua::PushBoolean(L, 0);
-        return 1;
-    }
-    Game::Lua::PushBoolean(L, 1);
+    Game::Lua::PushBoolean(L, itemID > 0);
     return 1;
 }
 

src/item/Info.cpp

@@ -93,13 +93,31 @@ static bool BuildIconPath(uint32_t displayInfoID, char *out, size_t outSize) {
     return true;
 }
 
+// `GetItemInfoInstant` is the modern "synchronous, never blocks" item
+// info call. In modern WoW the trailing 6 fields come from client-side
+// `Item-sparse.db2` and so the call always returns the full 7-tuple.
+// 1.12 has no client-side item DB — those fields live only in the
+// network-fed `DBCache_ItemStats`. We mirror the contract by:
+//   - always returning the itemID (echoed from input) so callers using
+//     `GetItemInfoInstant(link_or_id)` as an ID extractor never see nil
+//   - returning the remaining 6 fields from the cache when present,
+//     nil when not (preserving the 7-tuple shape)
+//   - not warming the cache; "Instant" implies no side effects. Callers
+//     that need the full info should use `GetItemInfo`, which already
+//     triggers the warmup via the `Script_GetItemInfo` hook.
 static int __fastcall Script_GetItemInfoInstant(void *L) {
     const int itemID = Item::Arg::ResolveItemID(L, 1);
     if (itemID <= 0)
         return 0;
+
+    Game::Lua::PushNumber(L, static_cast<double>(itemID));
+
     const uint8_t *record = FetchItemRecord(static_cast<uint32_t>(itemID));
-    if (record == nullptr)
-        return 0;
+    if (record == nullptr) {
+        for (int i = 0; i < 6; ++i)
+            Game::Lua::PushNil(L);
+        return 7;
+    }
 
     const uint32_t classID =
         *reinterpret_cast<const uint32_t *>(record + Offsets::OFF_ITEMSTATS_CLASS);
@@ -114,7 +132,6 @@ static int __fastcall Script_GetItemInfoInstant(void *L) {
     if (!BuildIconPath(displayInfoID, iconPath, sizeof(iconPath)))
         iconPath[0] = 0;
 
-    Game::Lua::PushNumber(L, static_cast<double>(itemID));
     Game::Lua::PushString(L, LookupItemClassName(classID));
     Game::Lua::PushString(L, LookupItemSubClassName(classID, subClassID));
     Game::Lua::PushString(L, LookupInvType(invType));

src/quest/Data.cpp

@@ -31,12 +31,10 @@ static const Event::Custom::AutoReserve _reserveQuestDataLoadResult{kQuestDataLo
 // `ret 8` cleans both args. We encode the questID into `userData` at
 // request time so the callback knows which quest completed.
 static void __stdcall QuestLoadCallback(void *userData, int success) {
-    const int slot = Event::Custom::Lookup(kQuestDataLoadResult);
-    if (slot < 0)
-        return;
     const auto questID =
         static_cast<int>(reinterpret_cast<uintptr_t>(userData));
-    Event::Custom::Fire(slot, "%d%d", questID, static_cast<int>(success != 0));
+    Event::Custom::FireIdSuccess(Event::Custom::Lookup(kQuestDataLoadResult),
+                                 questID, success != 0);
 }
 
 static void RequestAndMaybeNotify(uint32_t questID) {
@@ -47,9 +45,8 @@ static void RequestAndMaybeNotify(uint32_t questID) {
     // invoking our callback. Fire the event ourselves so addons get the
     // notification regardless of cache state, matching modern semantics.
     if (wasCached) {
-        const int slot = Event::Custom::Lookup(kQuestDataLoadResult);
-        if (slot >= 0)
-            Event::Custom::Fire(slot, "%d%d", static_cast<int>(questID), 1);
+        Event::Custom::FireIdSuccess(Event::Custom::Lookup(kQuestDataLoadResult),
+                                     static_cast<int>(questID), true);
     }
 }