redisq references -> r2 references

Author: cvweiss

DEBUGGING_ROUTING.md

@@ -6,7 +6,7 @@ Killmails are being posted to the wrong guildId/channelId despite the filter mat
 ## How the System Works
 
 ### Data Flow
-1. **poll-redisq.js** polls for new killmails
+1. **poll-r2.js** polls for new killmails
 2. For each killmail, it queries MongoDB for matching subscriptions
 3. Each subscription document contains: `{ guildId, channelId, entityIds, labels, iskValue, advanced }`
 4. When a match is found, it extracts `channelId` and `guildId` from the matched document
@@ -92,7 +92,7 @@ The `match` object is never mutated after being retrieved from MongoDB:
 
 ### Queue Processing
 ```javascript
-// poll-redisq.js - pushing to queue
+// poll-r2.js - pushing to queue
 const channelId = match.channelId;
 const guildId = match.guildId;
 discord_posts_queue.push({ db, match, guildId, channelId, killmail, zkb, colorCode, matchType });

Privacy.md

@@ -56,7 +56,7 @@ If the Bot is removed from a Discord server, all related configuration and histo
 
 zKillBot interacts with:
 - **EVE Online / CCP Games** — to retrieve killmail and entity information.
-- **https://zkillredisq.stream/** - [RedisQ](https://github.com/zKillboard/RedisQ) for polling of killmails that have been retrieved via https://zkillboard.com
+- **https://r2z2.zkillboard.com/sequence/*.json** zkill's R2 for polling of killmails via sequences
 - **Discord** — to send and receive data via the Discord API.
 
 All EVE Online intellectual property, including names, logos, and related materials, are the property of **CCP Games**.  

docs/index.html

@@ -334,7 +334,7 @@ <h5 class="card-title" style="color: #d9534f;">Enhanced Links</h5>
                     <div class="card-body">
                         <i class="bi bi-lightning-fill feature-icon"></i>
                         <h4 class="mb-3">Real-time Updates</h4>
-                        <p class="card-text">Receive killmails as they happen using zKillboard's RedisQ stream with minimal latency.</p>
+                        <p class="card-text">Receive killmails as they happen using zKillboard's R2 with minimal latency.</p>
                     </div>
                 </div>
             </div>
@@ -590,7 +590,7 @@ <h2 class="mb-4">Important Notes</h2>
             <div class="card">
                 <div class="card-body">
                     <ul>
-                        <li>zKillBot uses RedisQ to stream killmails from zKillBoard in near real-time.</li>
+                        <li>zKillBot uses R2 to poll killmails from zKillBoard in near real-time.</li>
                         <li>If a name query matches multiple results, zKillBot will list the IDs so you can subscribe by ID or refine your search.</li>
                         <li>A channel may have multiple subscriptions.</li>
                         <li>Each killmail will only be posted once per channel, even if it matches multiple subscriptions.</li>

env.example

@@ -1,5 +1,3 @@
-REDISQ_URL=https://zkillredisq.stream/listen.php?queueID=
-
 # Set to development and the bot will assign GUILD_ID too for instant slash command updates
 NODE_ENV=
 # Be sure to invite the bot to your server first.... (see INVITE below)

package.json

@@ -11,7 +11,7 @@
 	"keywords": [],
 	"author": "",
 	"license": "Unlicense",
-	"description": "A simple bot that listens to zKillboard's RedisQ for killmails and posts to Discord.",
+	"description": "A simple bot that listens to zKillboard's R2 for killmails and posts to Discord.",
 	"engines": {
 		"node": ">=22"
 	},

src/services/poll-redisq.js src/services/poll-r2.jssrc/services/poll-redisq.js renamed to src/services/poll-r2.js

@@ -9,47 +9,49 @@ let pollIntervalId = null;
 
 let errorCount = 0;
 
-export async function pollRedisQ(db, REDISQ_URL, sequence = 0) {
+export async function pollR2(db, sequence = 0) {
 	if (app_status.exiting) {
-		app_status.redisq_polling = false;
+		app_status.r2_polling = false;
 		if (pollIntervalId) {
 			clearTimeout(pollIntervalId);
 			pollIntervalId = null;
 		}
 		return;
 	}
 
-	let wait = 10000; // Default to being slow if there is no data
+	let wait = 6666; // Default to being slow if there is no data
 	let timer = null;
 	try {
 		if (discord_posts_queue.length > 100) {
-			// we'll wait the default timeout
+			wait = 1111; // If our queue is backed up, slow down polling to give it a chance to catch up
 			return;
 		}
 
 		if (sequence == 0) {
 			const row = await db.keyvalues.findOne({ key: "sequence" });
 			sequence = row?.value || 0;
 
-			// Ensure we have a valid starting sequence
-			const testRes = await fetch(`https://r2z2.zkillboard.com/ephemeral/${sequence}.json`, {
-				headers: HEADERS.headers
-			});
-			if (testRes.status === 404 && sequence > 0) {
-				console.log(`Stored RedisQ sequence ${sequence} is invalid, resetting to 0`);
-				sequence = 0;
+			if (sequence > 0) {
+				// Ensure we have a valid starting sequence
+				const testRes = await fetch(`https://r2z2.zkillboard.com/ephemeral/${sequence}.json`, {
+					headers: HEADERS.headers
+				});
+				if (testRes.status === 404 && sequence > 0) {
+					console.log(`Stored R2 sequence ${sequence} is invalid, resetting to 0`);
+					sequence = 0;
+				}
 			}
 
 			if (sequence == 0) {
 				const raw = await fetchWithRetry("https://r2z2.zkillboard.com/ephemeral/sequence.json");
 				const seqData = await raw.json();
 				sequence = seqData.sequence || 0;
 				if (sequence == 0) {
-					console.error("Failed to get initial RedisQ sequence number, defaulting to 0");
+					console.error("Failed to get initial R2 sequence number, defaulting to 0");
 					return;
 				}
 			}
-			console.log(`Starting RedisQ sequence at ${sequence}`);
+			console.log(`Starting R2 sequence at ${sequence}`);
 		}
 
 		let controller = new AbortController();
@@ -62,6 +64,7 @@ export async function pollRedisQ(db, REDISQ_URL, sequence = 0) {
 		clearTimeout(timer);
 		timer = null;
 		controller = null; // Release AbortController reference
+		console.log(new Date(), res.status, `https://r2z2.zkillboard.com/ephemeral/${sequence}.json`);
 
 		if (res.status !== 200) {
 			errorCount++;
@@ -252,11 +255,11 @@ export async function pollRedisQ(db, REDISQ_URL, sequence = 0) {
 				}
 			}
 
-			app_status.redisq_count++;
+			app_status.r2_count++;
 		}
 
 		if (data.sequence_id) {
-			wait = 50;
+			wait = 150;
 			sequence++;
 			await db.keyvalues.updateOne(
 				{ key: "sequence" },
@@ -268,13 +271,13 @@ export async function pollRedisQ(db, REDISQ_URL, sequence = 0) {
 		if (err.name === "AbortError") {
 			console.error("Fetch timed out after 15 seconds");
 		} else {
-			console.error("Error polling RedisQ:", err);
+			console.error("Error polling R2:", err);
 		}
 	} finally {
 		if (timer) {
 			clearTimeout(timer);
 			timer = null;
 		}
-		pollIntervalId = setTimeout(() => pollRedisQ(db, REDISQ_URL, sequence), wait);
+		pollIntervalId = setTimeout(() => pollR2(db, sequence), wait);
 	}
 }

src/util/app-status.js

@@ -1,23 +1,23 @@
 import { discord_posts_queue } from "../services/discord-post.js";
 
 export let app_status = {
-	redisq_count: 0,
+	r2_count: 0,
 	discord_post_count: 0,
-	redisq_polling: true,
+	r2_polling: true,
 	exiting: false
 };
 
 let statusIntervalId = null;
 
 export function shareAppStatus() {
 	const line = "📡" +
-		" RedisQ polls:".padEnd(20) + String(app_status.redisq_count).padStart(5) +
+		"R2 polls:".padEnd(20) + String(app_status.r2_count).padStart(5) +
 		"  Discord Queue:".padEnd(20) + String(discord_posts_queue.length).padStart(5) +
 		"  Discord Posts:".padEnd(20) + String(app_status.discord_post_count).padStart(5);
 
 	console.info(line);
 
-	app_status.redisq_count = 0;
+	app_status.r2_count = 0;
 	app_status.discord_post_count = 0;
 
 	if (statusIntervalId) clearTimeout(statusIntervalId);

src/util/constants.js

@@ -2,7 +2,7 @@ import { readFileSync } from 'fs';
 
 const pkg = JSON.parse(readFileSync("./package.json", "utf8"));
 const { name, version } = pkg;
-export const ZKILLBOT_VERSION = `${name} v${version}`;
+export const ZKILLBOT_VERSION = `${name} v${version}` + (process.env.NODE_ENV !== "production" ? " (dev)" : "");
 
 export const LOCALE = 'en';
 

src/util/shutdown.js

@@ -18,9 +18,9 @@ async function gracefulShutdown(signal) {
 
 		console.log(`⏹️ Preparing to shut down on ${signal}...`);
 
-		// wait for redisq_polling to finish and queue to drain (with 10s timeout)
+		// wait for r2 polling to finish and queue to drain (with 10s timeout)
 		const shutdownTimeout = Date.now() + 30000;
-		while ((app_status.redisq_polling || discord_posts_queue.length > 0) && Date.now() < shutdownTimeout) {
+		while ((app_status.r2_polling || discord_posts_queue.length > 0) && Date.now() < shutdownTimeout) {
 			await sleep(100);
 		}
 

src/zkillbot.js

@@ -17,7 +17,7 @@ import { writeHeapSnapshot } from "v8";
 // @ts-ignore
 dotenv.config({ quiet: true, path: new URL("../.env", import.meta.url).pathname });
 
-import { pollRedisQ } from "./services/poll-redisq.js";
+import { pollR2 } from "./services/poll-r2.js";
 
 import "./util/shutdown.js";
 
@@ -27,7 +27,7 @@ import { Db } from "mongodb";
 import { error } from "console";
 console.log(ZKILLBOT_VERSION);
 
-const { DISCORD_BOT_TOKEN, CLIENT_ID, MONGO_URI, MONGO_DB, REDISQ_URL } = process.env;
+const { DISCORD_BOT_TOKEN, CLIENT_ID, MONGO_URI, MONGO_DB } = process.env;
 export const { ZKILLBOT_CHANNEL_WEBHOOK } = process.env;
 
 // Memory debugging utilities
@@ -165,7 +165,7 @@ function startMemoryMonitoring() {
 	setTimeout(() => logMemoryUsage(), 5000);
 }
 
-if (!DISCORD_BOT_TOKEN || !CLIENT_ID || !REDISQ_URL || !MONGO_URI || !MONGO_DB) {
+if (!DISCORD_BOT_TOKEN || !CLIENT_ID || !MONGO_URI || !MONGO_DB) {
 	console.error("❌ Missing required env vars");
 	process.exit(1);
 }
@@ -233,7 +233,7 @@ async function init() {
 			// Start memory monitoring
 			//startMemoryMonitoring();
 
-			pollRedisQ(client.db, REDISQ_URL);
+			pollR2(client.db);
 			initCrons(client.db, client);
 		});