Log fix

.env.template

@@ -38,4 +38,8 @@ LOGDNA_DEFAULT_LEVEL=
 LOGDNA_TOKEN=
 
 # Sentry
-SENTRY_IO_DSN=
+SENTRY_IO_DSN=
+
+# Loki
+LOKI_USER_NAME=
+LOKI_PASSWORD=

.github/workflows/deploy-qa.yml

@@ -40,3 +40,5 @@ jobs:
           HD_NOTION_TOKEN: ${{secrets.QA_NOTION_TOKEN}}
           HD_LOGDNA_TOKEN: ${{secrets.QA_LOGDNA_TOKEN}}
           HD_SENTRY_IO_DSN: ${{secrets.QA_SENTRY_IO_DSN}}
+          HD_LOKI_USER_NAME: ${{secrets.QA_LOKI_USER_NAME}}
+          HD_LOKI_PASSWORD: ${{secrets.QA_LOKI_PASSWORD}}

CHANGELOG.md

@@ -1,9 +1,18 @@
 # Changelog
 
+<<<<<<< HEAD
+## 1.5-
+
+1. Updated github action workflow for qa to include loki
+2. Replaced logdna w/ pino-loki
+3. Integrate log aggregration with grafana
+4. Added loki auth fields to .env templates
+=======
 ## 1.5-PRE-RELEASE (2023-03-15)
 
 1. Bump json5 from 2.2.0 to 2.2.3 per Dependabot alert
 2. Bump webpack from 5.64.1 to 5.76.0 per Dependabot alert
+>>>>>>> develop
 
 ## 1.4.7-RELEASE (2022-12-30)
 

package.json

@@ -48,6 +48,8 @@
         "lodash.isequal": "^4.5.0",
         "mongodb": "^3.6.9",
         "p-queue": "^6.6.2",
+        "pino": "^8.11.0",
+        "pino-loki": "^2.0.4",
         "slash-create": "^5.4.1",
         "tslib": "^2.3.1",
         "uuid": "^8.3.2"

src/app/app.ts

@@ -92,15 +92,7 @@ function initializeDiscordEvents(): void {
 			}
 			Log.debug(`registered discordjs event ${event.name}`);
 		} catch (e) {
-			Log.error('Event failed to process', {
-				indexMeta: true,
-				meta: {
-					name: e.name,
-					message: e.message,
-					stack: e.stack,
-					event,
-				},
-			});
+			Log.error('Event failed to process');
 		}
 	});
 }
@@ -111,15 +103,6 @@ function initializeSlashCreateEvents(): void {
 	creator.on('error', (error: Error) => Log.error(`error: ${ error }`));
 	creator.on('synced', () => Log.debug('Commands synced!'));
 	creator.on('commandRegister', (command: SlashCommand) => Log.debug(`Registered command ${command.commandName}`));
-	creator.on('commandError', (command: SlashCommand, error: Error) => Log.error(`Command ${command.commandName}:`, {
-		indexMeta: true,
-		meta: {
-			name: error.name,
-			message: error.message,
-			stack: error.stack,
-			command,
-		},
-	}));
 	// Ran after the command has completed
 	creator.on('commandRun', (command:SlashCommand, result: Promise<any>, ctx: CommandContext) => {
 		LogUtils.logCommandEnd(ctx);
@@ -136,15 +119,7 @@ function initializeSlashCreateEvents(): void {
 			}
 			Log.debug(`Registered slash-create event ${event.name}`);
 		} catch (e) {
-			Log.error('Slash-create event failed to process', {
-				indexMeta: true,
-				meta: {
-					name: e.name,
-					message: e.message,
-					stack: e.stack,
-					event,
-				},
-			});
+			Log.error('Slash-create event failed to process');
 		}
 	});
 }

src/app/events/discordjs/GuildBanRemove.ts

@@ -14,14 +14,7 @@ export default class implements DiscordEvent {
 	async execute(ban: GuildBan): Promise<any> {
 		try {
 			if (ServiceUtils.isBanklessDAO(ban.guild)) {
-				Log.debug(`unbanning user: ${ban.user.tag}`, {
-					indexMeta: true,
-					meta: {
-						userId: ban.user.id,
-						userTag: ban.user.tag,
-						guildId: ban.guild.id,
-					},
-				});
+				Log.debug(`unbanning user: ${ban.user.tag}`);
 				// Add unbanned users to allowlist so they don't get auto-banned by the bot
 				const db: Db = await MongoDbUtils.connect(constants.DB_NAME_DEGEN);
 				const dbAllowlist = db.collection(constants.DB_COLLECTION_ALLOWLIST);

src/app/events/discordjs/RateLimit.ts

@@ -8,17 +8,7 @@ export default class implements DiscordEvent {
 
 	async execute(rateLimitData: RateLimitData): Promise<any> {
 		try {
-			Log.warn(`rate limit reached timeout: ${rateLimitData.timeout}`, {
-				indexMeta: true,
-				meta: {
-					timeout: rateLimitData.timeout,
-					limit: rateLimitData.limit,
-					method: rateLimitData.method,
-					path: rateLimitData.path,
-					route: rateLimitData.route,
-					global: rateLimitData.global,
-				},
-			});
+			Log.warn(`rate limit reached timeout: ${rateLimitData.timeout}`);
 		} catch (e) {
 			LogUtils.logError('failed to process event rateLimit', e);
 		}

src/app/events/discordjs/UserUpdate.ts

@@ -10,7 +10,7 @@ export default class implements DiscordEvent {
 	async execute(oldUser: User | PartialUser, newUser: User | PartialUser): Promise<any> {
 		try {
 			if (!oldUser || !newUser) {
-				Log.log('Skipping userUpdate event because oldUser or newUser was undefined.');
+				Log.info('Skipping userUpdate event because oldUser or newUser was undefined.');
 				return;
 			}
 

src/app/service/constants/apiKeys.ts

@@ -4,6 +4,8 @@ const apiKeys = Object.freeze({
 	logDNAToken: process.env.LOGDNA_TOKEN,
 	logDNAAppName: process.env.LOGDNA_APP_NAME,
 	logDNADefault: process.env.LOGDNA_DEFAULT_LEVEL,
+	lokiUserName: process.env.LOKI_USER_NAME,
+	lokiPassword: process.env.LOKI_PASSWORD,
 });
 
 export default apiKeys;

src/app/service/guest-pass/GuestPassService.ts

@@ -107,7 +107,7 @@ export default async (client: DiscordClient): Promise<void> => {
  * @param {string} tag Discord tag (e.g. hydrabolt#0001)
  */
 module.exports.findGuestPassPageByDiscordTag = async (tag: string): Promise<GetPageResponse> => {
-	Log.log('finding guest pass page by discord tag');
+	Log.info('finding guest pass page by discord tag');
 	const response = await notion.databases.query({
 		database_id: process.env.NOTION_GUEST_PASS_DATABASE_ID,
 		filter: {

src/app/service/timecard/Checkout.ts

@@ -45,15 +45,7 @@ export default async (guildMember: GuildMember, date: number, description: strin
 		throw new ValidationError('Event is not active.');
 	}
 
-	Log.info('timecard ended', {
-		indexMeta: true,
-		meta: {
-			discordId: activeTimecard.discordServerId,
-			duration: duration,
-			isActive: false,
-			description: description,
-		},
-	});
+	Log.info('timecard ended');
 
 	await guildMember.send(`Timecard finished at ${dayjs(date).format()}`);
 	return updateTimecardResult;

src/app/service/timecard/Hours.ts

@@ -27,12 +27,7 @@ export default async (guildMember: GuildMember): Promise<any> => {
 	const listOfTimeCards = await sendMultipleMessages(guildMember, completedTimeCards);
 
 
-	Log.info('Hours Requestsed', {
-		indexMeta: true,
-		meta: {
-			HoursReturned: 'completedTimeCards',
-		},
-	});
+	Log.info('Hours Requestsed');
 	return listOfTimeCards;
 };
 const sendMultipleMessages = async (guildMember: GuildMember, dbRecords: Cursor): Promise<any> => {

src/app/utils/Log.ts

@@ -1,150 +1,106 @@
-import logdna, { Logger, LogOptions } from '@logdna/logger';
+import { pino } from 'pino';
 import apiKeys from '../service/constants/apiKeys';
 import { CommandContext } from 'slash-create';
 import * as Sentry from '@sentry/node';
 
-let logger: Logger;
+const transport = pino.transport({
+	target: 'pino-loki',
+	options: {
+		batching: true,
+		interval: 5,
+		basicAuth: {
+			username: apiKeys.lokiUserName,
+			password: apiKeys.lokiPassword,
+		},
+	},
+});
 
-try {
-	logger = logdna.createLogger(apiKeys.logDNAToken, {
-		app: apiKeys.logDNAAppName,
-		level: apiKeys.logDNADefault,
-	});
-	if (process.env.NODE_ENV != 'production' || !logger.info) {
-		// eslint-disable-next-line no-console
-		console.log('Logger initialized!');
-	} else {
-		logger.log('Logger initialized!');
-	}
-} catch (e) {
-	// eslint-disable-next-line no-console
-	console.log('Please setup LogDNA token.');
-	// eslint-disable-next-line no-console
-	console.log(e);
-	throw new Error();
-}
+const logger = pino(transport);
 
 const Log = {
 
-	info(statement: string | any, options?: Omit<LogOptions, 'level'>): void {
+	info(statement: string | any): void {
 		if (process.env.NODE_ENV != 'production' || !logger.info) {
 			// eslint-disable-next-line no-console
 			console.log(statement);
 		} else {
-			logger.info(statement, options);
+			logger.info(statement);
 			Sentry.addBreadcrumb({
 				level: Sentry.Severity.Info,
 				message: statement,
 			});
 		}
 	},
 
-	warn(statement: string | any, options?: Omit<LogOptions, 'level'>): void {
+	warn(statement: string | any): void {
 		if (process.env.NODE_ENV != 'production' || !logger.warn) {
 			// eslint-disable-next-line no-console
 			console.log(statement);
 		} else {
-			logger.warn(statement, options);
+			logger.warn(statement);
 			Sentry.addBreadcrumb({
 				level: Sentry.Severity.Warning,
 				message: statement,
 			});
 		}
 	},
 
-	debug(statement: string | any, options?: Omit<LogOptions, 'level'>): void {
+	debug(statement: string | any): void {
 		if (process.env.NODE_ENV != 'production' || !logger.debug) {
 			// eslint-disable-next-line no-console
 			console.debug(statement);
 		} else {
-			logger.debug(statement, options);
+			logger.debug(statement);
 		}
 	},
 
-	error(statement: string | any, options?: Omit<LogOptions, 'level'>): void {
+	error(statement: string | any): void {
 		if (process.env.NODE_ENV != 'production' || !logger.error) {
 			// eslint-disable-next-line no-console
 			console.error(statement);
 		} else {
-			logger.error(statement, options);
+			logger.error(statement);
 			Sentry.addBreadcrumb({
 				level: Sentry.Severity.Error,
 				message: statement,
 			});
 		}
 	},
 
-	fatal(statement: string | any, options?: Omit<LogOptions, 'level'>): void {
+	fatal(statement: string | any): void {
 		if (process.env.NODE_ENV != 'production' || !logger.fatal) {
 			// eslint-disable-next-line no-console
 			console.error(statement);
 		} else {
-			logger.fatal(statement, options);
+			logger.fatal(statement);
 			Sentry.addBreadcrumb({
 				level: Sentry.Severity.Fatal,
 				message: statement,
 			});
 		}
 	},
 
-	trace(statement: string | any, options?: Omit<LogOptions, 'level'>): void {
+	trace(statement: string | any): void {
 		if (process.env.NODE_ENV != 'production' || !logger.trace) {
 			// eslint-disable-next-line no-console
 			console.log(statement);
 		} else {
-			logger.trace(statement, options);
+			logger.trace(statement);
 		}
 	},
 
-	log(statement: string | any, options?: Omit<LogOptions, 'level'>): void {
-		if (process.env.NODE_ENV != 'production') {
-			// eslint-disable-next-line no-console
-			console.log(statement);
-		}
-		logger.log(statement, options);
-		Sentry.addBreadcrumb({
-			level: Sentry.Severity.Log,
-			message: statement,
-		});
-	},
-
-	// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-	addMetaProperty(key: string, value: any): void {
-		logger.addMetaProperty(key, value);
-	},
-
-	removeMetaProperty(key: string): void {
-		logger.removeMetaProperty(key);
-	},
-
 	flush(): void {
 		logger.flush();
 	},
 };
 
 export const LogUtils = {
 	logCommandStart(ctx: CommandContext): void {
-		Log.info(`/${ctx.commandName} ran ${ctx.user.username}#${ctx.user.discriminator}`, {
-			indexMeta: true,
-			meta: {
-				guildId: ctx.guildID,
-				userTag: `${ctx.user.username}#${ctx.user.discriminator}`,
-				userId: ctx.user.id,
-				params: ctx.options,
-			},
-		});
+		Log.info(`/${ctx.commandName} ran ${ctx.user.username}#${ctx.user.discriminator}`);
 	},
 
 	logCommandEnd(ctx: CommandContext): void {
-		Log.info(`/${ctx.commandName} ended ${ctx.user.username}#${ctx.user.discriminator}`, {
-			indexMeta: true,
-			meta: {
-				guildId: ctx.guildID,
-				userTag: `${ctx.user.username}#${ctx.user.discriminator}`,
-				userId: ctx.user.id,
-				params: ctx.options,
-			},
-		});
+		Log.info(`/${ctx.commandName} ended ${ctx.user.username}#${ctx.user.discriminator}`);
 	},
 
 	logError(message: string, error: Error | any, guildId?: string): void {
@@ -154,15 +110,7 @@ export const LogUtils = {
 					guildId: guildId,
 				},
 			});
-			Log.error(message, {
-				indexMeta: true,
-				meta: {
-					name: error?.name,
-					message: error?.message,
-					stack: error?.stack,
-					guildId: guildId,
-				},
-			});
+			Log.error(message);
 		} catch (e) {
 			Log.error(message);
 		}