Add IotM tracking infrastructure (#300)

* Add IotM tracking: MrStore domain, database table, and live tracking

- New MrStore kol.js domain that parses mrstore.php for item name, descid,
  image, cost, currency, and category
- New Iotm database table with migration tracking item lifecycle dates
  (addedToStore, removedFromStore, distributedToSubscribers)
- Store check on rollover event to detect new/removed IotMs
- Subs webhook now records distributedToSubscribers date
- Refactor determineIotmMonthYear() to determineIotmMonth() returning a Date

* Rename IotM migration to timestamp format

Matches the naming convention of the flower_prices_created_at_index
migration that was added on main.

* Seed IotM table with historical data, make mraCost non-nullable

Migration now inserts 253 IotMs with names, descids, images, store
dates (from kol-exchange CSV), and 32 subscriber distribution dates
(from Discord channel history). mraCost is NOT NULL DEFAULT 1.

Author: gausie

packages/kol.js/src/domains/MrStore.test.ts

@@ -0,0 +1,51 @@
+import { expect, test } from "vitest";
+
+import { loadFixture } from "../testUtils.js";
+import { MrStore } from "./MrStore.js";
+
+test("Can parse Mr. Store items", async () => {
+  const page = await loadFixture(import.meta.dirname, "mrstore.html");
+
+  const items = MrStore.parse(page);
+
+  expect(items.length).toBeGreaterThan(0);
+
+  const iotm = items.find((i) => i.category === "April's Item-of-the-Month");
+  expect(iotm).toBeDefined();
+  expect(iotm).toMatchObject({
+    name: "wrapped Baseball Diamond",
+    descid: 844091703,
+    image: "bdiamond_box.gif",
+    cost: 1,
+    currency: "mr_accessory",
+    category: "April's Item-of-the-Month",
+  });
+});
+
+test("Parses item-of-the-year", async () => {
+  const page = await loadFixture(import.meta.dirname, "mrstore.html");
+
+  const items = MrStore.parse(page);
+
+  const ioty = items.find((i) => i.category === "2026's Item-of-the-Year");
+  expect(ioty).toMatchObject({
+    name: "discreetly-wrapped Eternity Codpiece",
+    descid: 323763723,
+    cost: 1,
+    currency: "mr_accessory",
+  });
+});
+
+test("Parses uncle buck items", async () => {
+  const page = await loadFixture(import.meta.dirname, "mrstore.html");
+
+  const items = MrStore.parse(page);
+
+  const ubItem = items.find((i) => i.currency === "uncle_buck");
+  expect(ubItem).toBeDefined();
+});
+
+test("Returns empty array for unparseable input", () => {
+  const items = MrStore.parse("");
+  expect(items).toEqual([]);
+});

packages/kol.js/src/domains/MrStore.ts

@@ -0,0 +1,54 @@
+import type { Client } from "../Client.js";
+
+export type MrStoreItem = {
+  name: string;
+  descid: number;
+  image: string;
+  cost: number;
+  currency: "mr_accessory" | "uncle_buck";
+  category: string;
+};
+
+export class MrStore {
+  #client: Client;
+
+  constructor(client: Client) {
+    this.#client = client;
+  }
+
+  async getCurrentItems(): Promise<MrStoreItem[]> {
+    const page = await this.#client.fetchText("mrstore.php");
+    return MrStore.parse(page);
+  }
+
+  static parse(page: string): MrStoreItem[] {
+    const boxPattern =
+      /<div\s+id=div\d+\s+width=320\s+class=mrbox>([\s\S]*?)<\/div>/g;
+
+    const items: MrStoreItem[] = [];
+
+    for (const boxMatch of page.matchAll(boxPattern)) {
+      const box = boxMatch[1];
+
+      const category = box.match(/<b style="color: white">(.+?)<\/b>/)?.[1];
+      const descid = box.match(/descitem\((\d+)\)/)?.[1];
+      const name = box.match(/class=nounder>(.+?)<\/a>/)?.[1];
+      const image = box.match(/itemimages\/(.+?\.gif)/)?.[1];
+      const cost = box.match(/<font size=\+1>(\d+)<\/font>/)?.[1];
+      const isUncleBuck = box.includes("unclebuck.gif");
+
+      if (!category || !descid || !name || !image || !cost) continue;
+
+      items.push({
+        name,
+        descid: Number(descid),
+        image,
+        cost: Number(cost),
+        currency: isUncleBuck ? "uncle_buck" : "mr_accessory",
+        category,
+      });
+    }
+
+    return items;
+  }
+}

packages/kol.js/src/domains/__fixtures__/mrstore.html

@@ -994,3 +994,131 @@ export async function getLatestDailies() {
   `.execute(db);
   return results.rows;
 }
+
+// ── Iotm ──
+
+export async function upsertIotmByName(data: {
+  itemName: string;
+  itemDescid?: number | null;
+  itemImage?: string | null;
+  month: Date;
+  mraCost?: number;
+  subscriberItem?: boolean;
+  addedToStore?: Date | null;
+}) {
+  // Check if there's an existing nameless subscriber row for this month we should claim
+  const existing = await db
+    .selectFrom("Iotm")
+    .selectAll()
+    .where("month", "=", data.month)
+    .where("subscriberItem", "=", data.subscriberItem ?? false)
+    .where("itemName", "is", null)
+    .executeTakeFirst();
+
+  if (existing) {
+    await db
+      .updateTable("Iotm")
+      .set({
+        itemName: data.itemName,
+        itemDescid: data.itemDescid ?? null,
+        itemImage: data.itemImage ?? null,
+        mraCost: data.mraCost,
+        addedToStore: data.addedToStore ?? null,
+      })
+      .where("id", "=", existing.id)
+      .execute();
+    return;
+  }
+
+  await db
+    .insertInto("Iotm")
+    .values({
+      itemName: data.itemName,
+      itemDescid: data.itemDescid ?? null,
+      itemImage: data.itemImage ?? null,
+      month: data.month,
+      ...(data.mraCost !== undefined && { mraCost: data.mraCost }),
+      subscriberItem: data.subscriberItem ?? false,
+      addedToStore: data.addedToStore ?? null,
+    })
+    .onConflict((oc) =>
+      oc.column("itemName").doUpdateSet({
+        itemDescid: data.itemDescid ?? null,
+        itemImage: data.itemImage ?? null,
+        ...(data.mraCost !== undefined && { mraCost: data.mraCost }),
+        addedToStore: data.addedToStore ?? null,
+      }),
+    )
+    .execute();
+}
+
+export async function upsertSubsRoll(month: Date, distributedAt: Date) {
+  const existing = await db
+    .selectFrom("Iotm")
+    .selectAll()
+    .where("month", "=", month)
+    .where("subscriberItem", "=", true)
+    .executeTakeFirst();
+
+  if (existing) {
+    await db
+      .updateTable("Iotm")
+      .set({ distributedToSubscribers: distributedAt })
+      .where("id", "=", existing.id)
+      .execute();
+    return;
+  }
+
+  await db
+    .insertInto("Iotm")
+    .values({
+      month,
+      subscriberItem: true,
+      distributedToSubscribers: distributedAt,
+    })
+    .execute();
+}
+
+export async function setIotmRemovedFromStore(
+  itemName: string,
+  removedAt: Date,
+) {
+  await db
+    .updateTable("Iotm")
+    .set({ removedFromStore: removedAt })
+    .where("itemName", "=", itemName)
+    .where("removedFromStore", "is", null)
+    .execute();
+}
+
+export async function getIotmsInStore() {
+  return await db
+    .selectFrom("Iotm")
+    .selectAll()
+    .where("addedToStore", "is not", null)
+    .where("removedFromStore", "is", null)
+    .where("itemName", "is not", null)
+    .execute();
+}
+
+export async function getIotmsForDateRange(from: Date, to: Date) {
+  return await db
+    .selectFrom("Iotm")
+    .selectAll()
+    .where((eb) =>
+      eb.or([
+        // Added to store within the range
+        eb.and([eb("addedToStore", ">=", from), eb("addedToStore", "<=", to)]),
+        // In store during the range (added before, removed after or still in)
+        eb.and([
+          eb("addedToStore", "<=", to),
+          eb.or([
+            eb("removedFromStore", "is", null),
+            eb("removedFromStore", ">=", from),
+          ]),
+        ]),
+      ]),
+    )
+    .orderBy("month", "asc")
+    .execute();
+}

packages/oaf/migrations/1776100000000_iotm.ts

@@ -0,0 +1,57 @@
+import { MrStore } from "kol.js/domains/MrStore";
+
+import {
+  getIotmsInStore,
+  setIotmRemovedFromStore,
+  upsertIotmByName,
+} from "../../clients/database.js";
+import { kolClient } from "../../clients/kol.js";
+import { determineIotmMonth } from "../../server/apps/oaf/routes/subs.js";
+
+const IOTM_CATEGORY_PATTERN = /^.+'s Item-of-the-Month$/;
+
+const mrStore = new MrStore(kolClient);
+
+async function checkStore() {
+  let items;
+  try {
+    items = await mrStore.getCurrentItems();
+  } catch {
+    return;
+  }
+
+  if (items.length === 0) return;
+
+  const now = new Date();
+  const month = determineIotmMonth();
+
+  const currentNames = new Set<string>();
+
+  for (const item of items) {
+    currentNames.add(item.name);
+
+    const subscriberItem = IOTM_CATEGORY_PATTERN.test(item.category);
+
+    await upsertIotmByName({
+      itemName: item.name,
+      itemDescid: item.descid,
+      itemImage: item.image,
+      month,
+      mraCost: item.cost,
+      subscriberItem,
+      addedToStore: now,
+    });
+  }
+
+  // Mark any previously-tracked items no longer in the store as removed
+  const inStore = await getIotmsInStore();
+  for (const iotm of inStore) {
+    if (iotm.itemName && !currentNames.has(iotm.itemName)) {
+      await setIotmRemovedFromStore(iotm.itemName, now);
+    }
+  }
+}
+
+export function init() {
+  kolClient.on("rollover", () => void checkStore());
+}

packages/oaf/src/clients/database.ts

@@ -105,6 +105,31 @@ export interface DailySubmissionTable {
   submittedAt: ColumnType<Date, Date | undefined, Date>;
 }
 
+export interface IotmTable {
+  id: Generated<number>;
+  itemName: string | null;
+  itemDescid: number | null;
+  itemImage: string | null;
+  month: ColumnType<Date, Date | string, Date | string>;
+  mraCost: Generated<number>;
+  subscriberItem: Generated<boolean>;
+  addedToStore: ColumnType<
+    Date | null,
+    Date | string | null,
+    Date | string | null
+  >;
+  removedFromStore: ColumnType<
+    Date | null,
+    Date | string | null,
+    Date | string | null
+  >;
+  distributedToSubscribers: ColumnType<
+    Date | null,
+    Date | string | null,
+    Date | string | null
+  >;
+}
+
 export interface DB {
   Player: PlayerTable;
   StandingOffer: StandingOfferTable;
@@ -119,6 +144,7 @@ export interface DB {
   FlowerPriceAlert: FlowerPriceAlertTable;
   Daily: DailyTable;
   DailySubmission: DailySubmissionTable;
+  Iotm: IotmTable;
 }
 
 export type Player = Selectable<PlayerTable>;
@@ -134,3 +160,4 @@ export type FlowerPrices = Selectable<FlowerPricesTable>;
 export type FlowerPriceAlert = Selectable<FlowerPriceAlertTable>;
 export type Daily = Selectable<DailyTable>;
 export type DailySubmission = Selectable<DailySubmissionTable>;
+export type Iotm = Selectable<IotmTable>;

packages/oaf/src/commands/kol/mrstore.ts

@@ -1,9 +1,9 @@
 import { afterEach, beforeEach } from "node:test";
 import { describe, expect, test, vi } from "vitest";
 
-import { determineIotmMonthYear } from "./subs.js";
+import { determineIotmMonth, formatIotmMonth } from "./subs.js";
 
-describe("Can parse dates correctly", () => {
+describe("determineIotmMonth", () => {
   beforeEach(() => {
     vi.useFakeTimers();
   });
@@ -14,16 +14,22 @@ describe("Can parse dates correctly", () => {
 
   test("Correctly rolls forward to the next month", () => {
     vi.setSystemTime(new Date(2023, 9, 27));
-    expect(determineIotmMonthYear()).toEqual("November 2023");
+    expect(determineIotmMonth()).toEqual(new Date(2023, 10, 1));
   });
 
   test("Correctly rolls backward to the current month", () => {
     vi.setSystemTime(new Date(2023, 9, 6));
-    expect(determineIotmMonthYear()).toEqual("October 2023");
+    expect(determineIotmMonth()).toEqual(new Date(2023, 9, 1));
   });
 
   test("Correctly rolls forward to the next year", () => {
     vi.setSystemTime(new Date(2023, 11, 30));
-    expect(determineIotmMonthYear()).toEqual("January 2024");
+    expect(determineIotmMonth()).toEqual(new Date(2024, 0, 1));
+  });
+});
+
+describe("formatIotmMonth", () => {
+  test("Formats a month correctly", () => {
+    expect(formatIotmMonth(new Date(2023, 9, 1))).toEqual("October 2023");
   });
 });