diff --git a/src/app/(general)/_components/landing-page/navbar/index.tsx b/src/app/(general)/_components/landing-page/navbar/index.tsx index 9f0e8fb2..338535c8 100644 --- a/src/app/(general)/_components/landing-page/navbar/index.tsx +++ b/src/app/(general)/_components/landing-page/navbar/index.tsx @@ -17,6 +17,9 @@ export const Navbar = () => { + + + diff --git a/src/app/(general)/_components/sidebar/main.tsx b/src/app/(general)/_components/sidebar/main.tsx index c0d61a05..72abc632 100644 --- a/src/app/(general)/_components/sidebar/main.tsx +++ b/src/app/(general)/_components/sidebar/main.tsx @@ -2,7 +2,7 @@ import Link from "next/link"; -import { Edit } from "lucide-react"; +import { BarChart3, Edit } from "lucide-react"; import { SidebarGroup, @@ -24,6 +24,11 @@ export const NavMain = () => { url: workbenchId ? `/workbench/${workbenchId}` : "/", icon: Edit, }, + { + title: "Models", + url: "/models", + icon: BarChart3, + }, ]; return ( diff --git a/src/app/(general)/models/page.tsx b/src/app/(general)/models/page.tsx new file mode 100644 index 00000000..4acf4562 --- /dev/null +++ b/src/app/(general)/models/page.tsx @@ -0,0 +1,163 @@ +import { Activity, Bot, Database, Gauge, Trophy } from "lucide-react"; + +import { Badge } from "@/components/ui/badge"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { HStack, VStack } from "@/components/ui/stack"; +import { api } from "@/trpc/server"; + +export const metadata = { + title: "Model Usage Leaderboard | Toolkit.dev", + description: + "Most-used language models across Toolkit.dev assistant messages.", +}; + +const numberFormatter = new Intl.NumberFormat("en-US"); + +export default async function ModelUsageLeaderboardPage() { + const { models, totalMessages } = await api.messages.getTopModelUsage({ + limit: 50, + }); + const maxCount = Math.max(...models.map((model) => model.count), 1); + + return ( +
+
+ + + + Rankings + +
+

+ Model usage leaderboard +

+

+ The most-used assistant models, ranked by messages generated in + Toolkit.dev. +

+
+ + + + {numberFormatter.format(totalMessages)} assistant messages + + +
+ +
+ + + + # + Model + Provider + Usage + Context + Capabilities + + + + {models.length === 0 ? ( + + + No model usage has been recorded yet. + + + ) : ( + models.map((model, index) => { + const width = `${(model.count / maxCount) * 100}%`; + + return ( + + + {index + 1} + + + +
+ +
+ + {model.name} + + {model.providerModelId} + + +
+
+ + + {model.provider} + + + + + + + {numberFormatter.format(model.count)} + + {model.percentage}% + + +
+
+
+ + + + {model.contextLength ? ( + + + + {numberFormatter.format(model.contextLength)} + + + ) : ( + + Unknown + + )} + + + + {model.capabilities.length > 0 ? ( + model.capabilities.map((capability) => ( + + {capability.replaceAll("-", " ")} + + )) + ) : ( + + None listed + + )} + + + + ); + }) + )} + +
+
+
+
+ ); +} diff --git a/src/server/api/routers/messages.ts b/src/server/api/routers/messages.ts index 1522962f..11950b87 100644 --- a/src/server/api/routers/messages.ts +++ b/src/server/api/routers/messages.ts @@ -1,7 +1,12 @@ import { z } from "zod"; -import { createTRPCRouter, protectedProcedure } from "@/server/api/trpc"; +import { + createTRPCRouter, + protectedProcedure, + publicProcedure, +} from "@/server/api/trpc"; import { FILE_NAME_MAX_LENGTH } from "@/lib/constants"; +import { languageModels } from "@/ai/language"; const messagePartSchema = z.discriminatedUnion("type", [ z.object({ @@ -43,6 +48,58 @@ const messagePartSchema = z.discriminatedUnion("type", [ }), ]); export const messagesRouter = createTRPCRouter({ + getTopModelUsage: publicProcedure + .input( + z + .object({ + limit: z.number().min(1).max(100).default(25), + }) + .optional(), + ) + .query(async ({ ctx, input }) => { + const limit = input?.limit ?? 25; + + const [modelRows, totalMessages] = await Promise.all([ + ctx.db.message.groupBy({ + by: ["modelId"], + where: { + role: "assistant", + }, + _count: { + _all: true, + }, + orderBy: { + _count: { + modelId: "desc", + }, + }, + take: limit, + }), + ctx.db.message.count({ + where: { + role: "assistant", + }, + }), + ]); + + return { + totalMessages, + models: modelRows.map((row) => { + const resolvedModel = resolveModel(row.modelId); + const count = row._count._all; + + return { + ...resolvedModel, + count, + percentage: + totalMessages === 0 + ? 0 + : Math.round((count / totalMessages) * 1000) / 10, + }; + }), + }; + }), + getMessagesForChat: protectedProcedure .input( z.object({ @@ -138,3 +195,53 @@ export const messagesRouter = createTRPCRouter({ }); }), }); + +function resolveModel(storedModelId: string) { + const separatorIndex = getModelSeparatorIndex(storedModelId); + const provider = + separatorIndex > 0 ? storedModelId.slice(0, separatorIndex) : "unknown"; + const providerModelId = + separatorIndex > 0 + ? storedModelId.slice(separatorIndex + 1) + : storedModelId; + const model = languageModels.find( + (model) => + model.provider === provider && + (model.modelId === providerModelId || + `${model.provider}/${model.modelId}` === storedModelId || + `${model.provider}:${model.modelId}` === storedModelId), + ); + + return { + modelId: storedModelId, + provider, + providerModelId, + name: model?.name ?? formatModelName(providerModelId), + description: model?.description, + contextLength: model?.contextLength, + capabilities: model?.capabilities ?? [], + }; +} + +function getModelSeparatorIndex(modelId: string) { + const slashIndex = modelId.indexOf("/"); + const colonIndex = modelId.indexOf(":"); + + if (slashIndex === -1) { + return colonIndex; + } + + if (colonIndex === -1) { + return slashIndex; + } + + return Math.min(slashIndex, colonIndex); +} + +function formatModelName(modelId: string) { + return modelId + .split(/[-_:/.]+/) + .filter(Boolean) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +}