diff --git a/README.md b/README.md index 922e4014..22f8e388 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ NODE_EXTRA_CA_CERTS=$PWD/otterscale-ca.crt pnpm dev --host Setup, configuration, and deployment guides will be published in the project documentation. See `.env.example` for the environment variables the app expects. +- [Metrics in the dashboard](docs/metrics.md): which metrics a page may use, and why regular users see less. + ## Ecosystem OtterScale Dashboard is the web UI for the OtterScale platform. See the [otterscale](https://github.com/otterscale/otterscale) repository for an overview of the full project and its components. diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 00000000..8fd69f25 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,19 @@ +# Metrics in the dashboard + +The OtterScale server answers metrics at three scopes; the shared one takes no PromQL: + +| Scope | Path | Who | What | +| --------- | ---------------------------------------------------- | ------------------------------------------------------------ | -------------------------------------------------------------------------------- | +| Cluster | `/proxy//prometheus` | cluster-admin, or bound to `otterscale:metrics-cluster-view` | any query | +| Namespace | `/proxy//prometheus-tenancy?namespace=` | anyone who may `get pods` in `` | any query, rewritten to `namespace=""` the way prom-label-proxy does it | +| Shared | `/proxy//prometheus-shared/` | the namespace scope's callers, and cluster scope | one of the server's named queries: per GPU card, per host, or a total; no PromQL | + +Everyone else gets 403 at cluster scope, whatever the query. Create drivers with `clusterPrometheusDriver`, `namespacePrometheusDriver` or `workspacePrometheusDriver` (`src/lib/prometheus.ts`), never `new PrometheusDriver` with a hand-written endpoint. `page.data.canViewClusterMetrics` says which scope the viewer has. Use `sharedMetrics(cluster, namespace)` for the shared scope; `namespace` is the Workspace's when there is one, never an admin's namespace choice, and is left out otherwise, which only cluster viewers may do. + +## Rules for namespace-scoped pages + +- Use only metrics whose `namespace` label is the tenant's namespace: kube-state-metrics, cAdvisor and kubelet volume metrics, vLLM, KubeVirt VMI metrics (`namespace`, not `exported_namespace`). +- Node, GPU (DCGM, HAMi), Ceph and KubeVirt component metrics are infrastructure: their `namespace` is the exporter's, and their series can name other tenants' pods. On a Workspace page, take these numbers from a shared query (`SharedQueryName` lists them) for every viewer, admins included, so both see the same numbers. If no query fits, the new one goes into the server's catalog (`internal/handler/proxy_shared_catalog.go`), behind its safety test and a security review — never PromQL sent from here. Otherwise hide the component when `canViewClusterMetrics` is false: no empty chart, no 403. +- Do not select on `namespace` yourself with `=~`, `!=` or `!~` against other namespaces: the server refuses a selector that cannot match the enforced namespace with 400. +- Build an optional namespace filter with `namespaceSelector(ns)` (`src/lib/prometheus.ts`): it escapes the value and leaves the matcher out for an admin's empty "all namespaces" choice, where `namespace=""` would match only series without the label. +- Testing with an admin account proves nothing here: an admin keeps cluster scope. Run `e2e/tenant-metrics.test.ts` as a regular Workspace member (see the comment at its top). diff --git a/e2e/tenant-metrics.test.ts b/e2e/tenant-metrics.test.ts new file mode 100644 index 00000000..51632bae --- /dev/null +++ b/e2e/tenant-metrics.test.ts @@ -0,0 +1,350 @@ +/* + * A regular user's dashboard: what they see, and that the charts carry data + * from their own namespace only. Runs against a deployed dashboard: + * + * E2E_BASE_URL=https:// E2E_IGNORE_HTTPS_ERRORS=1 \ + * E2E_TENANT_USERNAME=... E2E_TENANT_PASSWORD=... \ + * E2E_CLUSTER= E2E_WORKSPACE= E2E_NAMESPACE= \ + * [E2E_MODEL=] \ + * pnpm exec playwright test e2e/tenant-metrics.test.ts + * + * The user must be a plain Workspace member (view is enough): not cluster-admin, + * not bound to otterscale:metrics-cluster-view, in no Keycloak group that + * grants either. An admin account proves nothing here — admins keep cluster + * scope. E2E_EXPECT_VM_DATA / E2E_EXPECT_MODEL_DATA=1 also require data on the + * compute / model pages (set them when the namespace runs a VM / a model); + * E2E_MODEL names the model for the analytics and model-service tests, which + * are skipped without it. E2E_OTHER_NAMESPACE (default kube-system) is a + * namespace the user has no access to, for the server's own refusals. + */ +import { expect, type Page, test } from '@playwright/test'; + +const env = { + username: process.env.E2E_TENANT_USERNAME ?? '', + password: process.env.E2E_TENANT_PASSWORD ?? '', + cluster: process.env.E2E_CLUSTER ?? '', + workspace: process.env.E2E_WORKSPACE ?? '', + namespace: process.env.E2E_NAMESPACE ?? '' +}; + +/** Optional: the model whose analytics and actions are checked. */ +const model = process.env.E2E_MODEL ?? ''; + +/** A namespace the user may not see, for asking the server what the dashboard never asks. */ +const otherNamespace = process.env.E2E_OTHER_NAMESPACE ?? 'kube-system'; + +/** The server's named queries (otterscale internal/handler/proxy_shared_catalog.go). */ +const SHARED_NAMES = new Set([ + 'gpu-util', + 'gpu-fb-used', + 'gpu-fb-total', + 'gpu-inventory', + 'vgpu-memory-limit', + 'vgpu-memory-used', + 'host-gpu-fb-used', + 'host-gpu-fb-total', + 'host-gpu-count', + 'gpu-xid-total', + 'gpu-xid-by-gpu', + 'kubevirt-handlers', + 'kubevirt-controllers', + 'kubevirt-controllers-ready', + 'ceph-total-bytes', + 'gpu-allocated-memory', + 'gpu-allocated-cores', + 'gpu-shared-count' +]); + +test.skip( + !process.env.E2E_BASE_URL || Object.values(env).some((v) => !v), + 'needs E2E_BASE_URL and the E2E_TENANT_* / E2E_CLUSTER / E2E_WORKSPACE / E2E_NAMESPACE variables' +); +// Two-step Keycloak login plus a dashboard that queries after mount: the +// 30 s default has timed out once in beforeEach alone. +test.describe.configure({ timeout: 120_000 }); + +type Sample = { + metric?: Record; + value?: [number, string]; + values?: [number, string][]; +}; +type Captured = { url: URL; status: number; result: Sample[] }; +type SharedCall = { name: string; status: number; params: URLSearchParams }; + +/** Infrastructure metrics: never in a namespace-scoped query; the shared scope serves them by name. */ +const INFRASTRUCTURE = [ + 'ceph_cluster_total_bytes', + 'kubevirt_info', + 'kubevirt_virt_controller_ready_status', + 'DCGM_FI_DEV_FB_USED', + 'DCGM_FI_DEV_XID_ERRORS' +]; + +function watchMetrics(page: Page) { + const tenancy: Captured[] = []; + const shared: SharedCall[] = []; + const clusterScope: string[] = []; + const sharedPrefix = `/proxy/${env.cluster}/prometheus-shared/`; + page.on('request', (req) => { + if (req.url().includes(`/proxy/${env.cluster}/prometheus/`)) clusterScope.push(req.url()); + }); + page.on('response', async (res) => { + const url = new URL(res.url()); + if (url.pathname.startsWith(sharedPrefix)) { + shared.push({ + name: url.pathname.slice(sharedPrefix.length), + status: res.status(), + params: url.searchParams + }); + return; + } + if (!url.pathname.startsWith(`/proxy/${env.cluster}/prometheus-tenancy/`)) return; + let result: Sample[] = []; + try { + const body = await res.json(); + result = Array.isArray(body?.data?.result) ? body.data.result : []; + } catch { + // label answers are not vectors + } + tenancy.push({ url, status: res.status(), result }); + }); + return { tenancy, shared, clusterScope }; +} + +/** + * Wait for an answer to each named shared query. Start it before the action + * that sends them, so the answers it waits for come after that action. + */ +function sharedAnswers(page: Page, names: string[]) { + const path = (name: string) => `/proxy/${env.cluster}/prometheus-shared/${name}`; + return Promise.all( + names.map((name) => page.waitForResponse((res) => new URL(res.url()).pathname === path(name))) + ); +} + +/** Every shared call is a known name, answered, asked as this Workspace; `names` were all asked. */ +function expectShared(calls: SharedCall[], names: string[]) { + for (const c of calls) { + expect(SHARED_NAMES.has(c.name), `unknown shared query ${c.name}`).toBe(true); + expect(c.status, `${c.name} answered ${c.status}`).toBeLessThan(400); + expect(c.params.getAll('namespace')).toEqual([env.namespace]); + } + const asked = new Set(calls.map((c) => c.name)); + for (const name of names) expect(asked.has(name), `${name} was not asked`).toBe(true); +} + +const sentQueries = (captured: Captured[]) => + captured.map((c) => + [c.url.searchParams.get('query') ?? '', ...c.url.searchParams.getAll('match[]')].join(' ') + ); + +function hasData(captured: Captured[]) { + return captured.some((c) => + c.result.some((s) => { + const v = Number(s.value?.[1] ?? s.values?.at(-1)?.[1]); + return Number.isFinite(v) && v !== 0; + }) + ); +} + +function expectOwnNamespaceOnly(captured: Captured[]) { + expect(captured.length, 'no namespace-scoped query was sent').toBeGreaterThan(0); + for (const c of captured) { + expect(c.status, `${c.url.pathname} answered ${c.status}`).toBeLessThan(400); + expect(c.url.searchParams.getAll('namespace')).toEqual([env.namespace]); + for (const s of c.result) { + if (s.metric?.namespace !== undefined) expect(s.metric.namespace).toBe(env.namespace); + } + } +} + +async function login(page: Page) { + await page.goto('/login'); + // The otterscale realm asks for the username first, then the password on a second page. + await page.locator('#username').fill(env.username); + await page.locator('#kc-login').click(); + await page.locator('#password').fill(env.password); + await page.locator('#kc-login').click(); + // Back on the dashboard: not on another origin's Keycloak, nor on one mounted under /auth/, + // nor still on /login/callback. + await page.waitForURL( + (url) => + url.origin === new URL(process.env.E2E_BASE_URL ?? '').origin && + !/^\/(auth|login)(\/|$)/.test(url.pathname), + { timeout: 60_000 } + ); +} + +const dashboard = (name: string) => `/${env.cluster}/${env.workspace}/dashboard/${name}`; + +/** + * Open a dashboard and wait until its widgets, which query after mount, have + * been answered: at least one namespace-scoped query, and each of `sharedNames`. + */ +async function visit(page: Page, name: string, sharedNames: string[] = []) { + const watched = watchMetrics(page); + const answered = sharedAnswers(page, sharedNames); + await page.goto(dashboard(name)); + await answered; + await expect + .poll(() => watched.tenancy.length, { message: 'no namespace-scoped query' }) + .toBeGreaterThan(0); + await page.waitForLoadState('networkidle'); + return watched; +} + +test.beforeEach(async ({ page }) => { + await login(page); +}); + +test('the sidebar has no cluster or storage dashboard', async ({ page }) => { + await page.goto(dashboard('overview')); + await page.waitForLoadState('networkidle'); + // The sidebar has rendered once the model link is in the DOM; only then does a + // missing link mean anything. Attached, not visible: on the overview its group + // is collapsed, and sub-items of a closed group are in the DOM but hidden. + await expect(page.locator(`a[href$="/dashboard/model"]`).first()).toBeAttached(); + await expect(page.locator(`a[href$="/dashboard/cluster"]`)).toHaveCount(0); + await expect(page.locator(`a[href$="/dashboard/storage"]`)).toHaveCount(0); +}); + +for (const name of ['cluster', 'storage']) { + test(`opening the ${name} dashboard directly lands on the overview and says why`, async ({ + page + }) => { + await page.goto(dashboard(name)); + await page.waitForURL( + (url) => url.pathname.endsWith('/dashboard/overview') && !url.searchParams.has('notice') + ); + await expect(page.getByText(/cluster metrics access|叢集指標權限/)).toBeVisible(); + }); +} + +test('the server refuses cluster scope, another namespace, and shared without a namespace', async ({ + page +}) => { + expect(otherNamespace).not.toBe(env.namespace); + // Straight to the server through the dashboard's proxy, as the page's own requests go. + const ask = (path: string, params: Record = {}) => + page.request.get(`/proxy/${env.cluster}/${path}`, { + headers: { 'x-proxy-target': 'api' }, + params + }); + + const cluster = await ask('prometheus/api/v1/query', { query: 'up' }); + expect(cluster.status(), 'cluster scope').toBe(403); + const tenancy = await ask('prometheus-tenancy/api/v1/query', { + query: 'up', + namespace: otherNamespace + }); + expect(tenancy.status(), `namespace scope in ${otherNamespace}`).toBe(403); + const shared = await ask('prometheus-shared/ceph-total-bytes'); + expect(shared.status(), 'shared scope without a namespace').toBe(403); + // The same shared query as this Workspace is answered: the 403 above is the namespace's doing. + const own = await ask('prometheus-shared/ceph-total-bytes', { namespace: env.namespace }); + expect(own.status(), 'shared scope as this Workspace').toBe(200); +}); + +test('overview shows their namespace, and the cluster capacity through the shared scope', async ({ + page +}) => { + const { tenancy, shared, clusterScope } = await visit(page, 'overview', ['ceph-total-bytes']); + expectOwnNamespaceOnly(tenancy); + await expect.poll(() => hasData(tenancy), { message: 'overview charts are empty' }).toBe(true); + expect(clusterScope).toEqual([]); + for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); + expectShared(shared, ['ceph-total-bytes']); +}); + +test('compute stays in their namespace and shows KubeVirt components', async ({ page }) => { + const kubevirt = ['kubevirt-handlers', 'kubevirt-controllers', 'kubevirt-controllers-ready']; + const { tenancy, shared, clusterScope } = await visit(page, 'compute', kubevirt); + expectOwnNamespaceOnly(tenancy); + expect(clusterScope).toEqual([]); + for (const q of sentQueries(tenancy)) { + for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); + expect(q).not.toContain('exported_namespace'); + } + expectShared(shared, kubevirt); + if (process.env.E2E_EXPECT_VM_DATA === '1') + await expect.poll(() => hasData(tenancy), { message: 'compute charts are empty' }).toBe(true); +}); + +test('model overview stays in their namespace', async ({ page }) => { + const { tenancy, shared, clusterScope } = await visit(page, 'model'); + expectOwnNamespaceOnly(tenancy); + expect(clusterScope).toEqual([]); + for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); + expectShared(shared, []); + if (process.env.E2E_EXPECT_MODEL_DATA === '1') + await expect.poll(() => hasData(tenancy), { message: 'model charts are empty' }).toBe(true); +}); + +test('model analytics shows GPU memory and XID through the shared scope', async ({ page }) => { + test.skip(!model, 'needs E2E_MODEL'); + const watched = watchMetrics(page); + const gpuNames = [ + 'host-gpu-fb-used', + 'host-gpu-fb-total', + 'host-gpu-count', + 'gpu-xid-total', + 'gpu-xid-by-gpu' + ]; + // The panels query once a model is picked; started before any click, this + // also catches answers for a model the page picked on its own. + const answered = sharedAnswers(page, gpuNames); + await page.goto(dashboard('model')); + await page.getByRole('tab', { name: /^(Analytics|分析)$/ }).click(); + await page.getByRole('combobox').first().click(); + await page.getByRole('option', { name: model, exact: true }).click(); + await answered; + + await expect( + page.getByText(/GPU Memory: Allocated vs Used|GPU 記憶體：已分配 vs 已使用/) + ).toBeVisible(); + await expect(page.getByText('XID', { exact: true })).toBeVisible(); + expectOwnNamespaceOnly(watched.tenancy); + expect(watched.clusterScope).toEqual([]); + expectShared(watched.shared, gpuNames); +}); + +test('a model service shows its GPU allocation and the GPU resources', async ({ page }) => { + test.skip(!model, 'needs E2E_MODEL'); + const watched = watchMetrics(page); + const list = `/${env.cluster}/${env.workspace}?group=serving.kserve.io&version=v1alpha2&kind=LLMInferenceService&resource=llminferenceservices`; + const openActions = async () => { + await page + .getByRole('row', { name: new RegExp(model) }) + .getByRole('button') + .last() + .click(); + }; + + await page.goto(list); + await openActions(); + const allocationNames = [ + 'gpu-util', + 'gpu-fb-used', + 'gpu-fb-total', + 'gpu-allocated-memory', + 'gpu-shared-count' + ]; + const allocationAnswered = sharedAnswers(page, allocationNames); + await page.getByRole('menuitem', { name: /^(GPU Allocation|GPU 分配)$/ }).click(); + await expect(page.getByRole('dialog')).toBeVisible(); + await allocationAnswered; + expectShared(watched.shared, allocationNames); + await page.keyboard.press('Escape'); + + await openActions(); + // The GPU resources ask as the Update dialog opens, before their button is clicked. + const resourceNames = ['gpu-inventory', 'vgpu-memory-limit', 'vgpu-memory-used']; + const resourcesAnswered = sharedAnswers(page, resourceNames); + await page.getByRole('menuitem', { name: /^Update$/ }).click(); + // The row behind the dialog has GPU icons too; the overlay would take the click. + await page.getByRole('dialog').last().locator('button:has(svg.lucide-gpu)').first().click(); + await expect(page.getByText('GPU Resources', { exact: true })).toBeVisible(); + await resourcesAnswered; + expectShared(watched.shared, resourceNames); + expect(watched.clusterScope).toEqual([]); +}); diff --git a/messages/en.json b/messages/en.json index 1a828873..3d292779 100644 --- a/messages/en.json +++ b/messages/en.json @@ -176,6 +176,7 @@ "gpu_allocation_node_description": "Pod → GPU → Node relationship diagram", "gpu_allocation_loading": "Loading GPU allocation...", "gpu_allocation_fetch_error": "Failed to fetch GPU allocation: {message}", + "gpu_resources_fetch_error": "Couldn't load GPU resources: {message}", "gpu_allocation_empty_model": "No GPU devices found for this model. Pods may not have HAMi vGPU allocations.", "gpu_allocation_empty_node": "No GPU devices registered on this node.", "node_mig_enable_alert_title": "This feature is not recommended", @@ -410,6 +411,9 @@ "wait": "Wait", "delay": "Delay", "no_data_display": "No data to display", + "cluster_metrics_access_unknown": "Couldn't confirm your cluster metrics access, so the Cluster and Storage dashboards are hidden from the menu for now. Refresh the page to try again.", + "cluster_metrics_required": "The Cluster and Storage dashboards need cluster metrics access. Ask your administrator for it.", + "workspace_namespace_unknown": "Couldn't find this workspace's namespace, so there are no metrics to show. Refresh the page, or ask your administrator to check the workspace.", "no_traffic_display": "No requests in this range", "no_traffic_hint": "The model is running but received no traffic.", "no_data_available": "No data available", @@ -638,6 +642,8 @@ "metric_restart": "Restart", "metric_oom": "OOM", "metric_xid": "XID", + "xid_unavailable": "Couldn't load XID errors right now. Refresh to try again.", + "pod_health_unavailable": "Couldn't load pod health right now. Refresh to try again.", "metric_cpu_throttle": "Throttle", "metric_shm_crash": "Shm", "metric_preempt": "Preempt", diff --git a/messages/zh-hant.json b/messages/zh-hant.json index 905e468a..d4b01db5 100644 --- a/messages/zh-hant.json +++ b/messages/zh-hant.json @@ -176,6 +176,7 @@ "gpu_allocation_node_description": "Pod → GPU → Node 關聯圖", "gpu_allocation_loading": "正在載入 GPU 分配資訊...", "gpu_allocation_fetch_error": "無法取得 GPU 分配資訊：{message}", + "gpu_resources_fetch_error": "無法取得 GPU 資源：{message}", "gpu_allocation_empty_model": "此模型未找到 GPU 裝置。Pod 可能沒有 HAMi vGPU 分配。", "gpu_allocation_empty_node": "此節點上沒有已註冊的 GPU 裝置。", "node_mig_enable_alert_title": "不建議使用此功能", @@ -410,6 +411,9 @@ "wait": "等待", "delay": "延遲", "no_data_display": "尚無資料可顯示", + "cluster_metrics_access_unknown": "暫時無法確認叢集指標權限，叢集與儲存儀表板先從選單隱藏。請重新整理頁面再試一次。", + "cluster_metrics_required": "叢集與儲存儀表板需要叢集指標權限，請洽管理員。", + "workspace_namespace_unknown": "找不到這個工作區的命名空間，沒有可以顯示的指標。請重新整理頁面，或請管理員檢查工作區設定。", "no_traffic_display": "此區間內沒有請求", "no_traffic_hint": "模型運行中,但沒有收到流量。", "no_data_available": "目前無可用資料", @@ -638,6 +642,8 @@ "metric_restart": "重啟", "metric_oom": "OOM", "metric_xid": "XID", + "xid_unavailable": "暫時無法取得 XID 錯誤，請重新整理再試一次。", + "pod_health_unavailable": "暫時無法取得 Pod 狀態，請重新整理再試一次。", "metric_cpu_throttle": "節流", "metric_shm_crash": "Shm", "metric_preempt": "搶占", diff --git a/package.json b/package.json index 991f776c..02178439 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,10 @@ "brace-expansion@<5.0.12": "5.0.12", "marked": "^18.0.7", "devalue@<5.9.3": "5.9.3", - "postcss@<8.5.28": "8.5.28" + "postcss@<8.5.28": "8.5.28", + "source-map-js@<1.2.2": "1.2.2", + "postcss-selector-parser@<7.1.6": "7.1.6", + "vite-plugin-static-copy>chokidar": "^4.0.3" }, "onlyBuiltDependencies": [ "esbuild", diff --git a/playwright.config.ts b/playwright.config.ts index f6c81af8..f7a28689 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,9 +1,22 @@ import { defineConfig } from '@playwright/test'; +// E2E_BASE_URL runs the suite against a deployed dashboard (e2e/tenant-metrics +// needs a real Keycloak user and hub); without it, against a local preview. +const baseURL = process.env.E2E_BASE_URL; + export default defineConfig({ - webServer: { - command: 'npm run build && npm run preview', - port: 4173 - }, - testDir: 'e2e' + testDir: 'e2e', + ...(baseURL + ? { + use: { + baseURL, + ignoreHTTPSErrors: process.env.E2E_IGNORE_HTTPS_ERRORS === '1' + } + } + : { + webServer: { + command: 'npm run build && npm run preview', + port: 4173 + } + }) }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 91f5fbb5..66115e0a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,9 @@ overrides: marked: ^18.0.7 devalue@<5.9.3: 5.9.3 postcss@<8.5.28: 8.5.28 + source-map-js@<1.2.2: 1.2.2 + postcss-selector-parser@<7.1.6: 7.1.6 + vite-plugin-static-copy>chokidar: ^4.0.3 importers: @@ -1503,10 +1506,6 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} - anymatch@3.1.3: - resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} - engines: {node: '>= 8'} - arctic@3.7.0: resolution: {integrity: sha512-ZMQ+f6VazDgUJOd+qNV+H7GohNSYal1mVjm5kEaZfE2Ifb7Ss70w+Q7xpJC87qZDkMZIXYf0pTIYZA0OPasSbw==} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. @@ -1545,10 +1544,6 @@ packages: bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} - binary-extensions@2.3.0: - resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} - engines: {node: '>=8'} - bits-ui@2.18.1: resolution: {integrity: sha512-KkemzKFH4T3gt3H+P86JcnAWExjByv/6vlwjm/BoCwTPHu03yiCdxbghdJLvFReQTe0acCAiRcKfmixxD6XvlA==} engines: {node: '>=20'} @@ -1560,10 +1555,6 @@ packages: resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} engines: {node: 20 || >=22} - braces@3.0.3: - resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} - engines: {node: '>=8'} - call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -1584,10 +1575,6 @@ packages: character-entities@2.0.2: resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} - chokidar@3.6.0: - resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} - engines: {node: '>= 8.10.0'} - chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -2036,10 +2023,6 @@ packages: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} - fill-range@7.1.1: - resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} - engines: {node: '>=8'} - find-up@5.0.0: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} @@ -2085,10 +2068,6 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} - glob-parent@5.1.2: - resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} - engines: {node: '>= 6'} - glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -2181,10 +2160,6 @@ packages: resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==} engines: {node: '>=12.22.0'} - is-binary-path@2.1.0: - resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} - engines: {node: '>=8'} - is-core-module@2.16.1: resolution: {integrity: sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==} engines: {node: '>= 0.4'} @@ -2200,10 +2175,6 @@ packages: is-module@1.0.0: resolution: {integrity: sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==} - is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} @@ -2606,10 +2577,6 @@ packages: resolution: {integrity: sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==} hasBin: true - normalize-path@3.0.0: - resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} - engines: {node: '>=0.10.0'} - obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} @@ -2660,10 +2627,6 @@ packages: picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - picomatch@2.3.2: - resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} - engines: {node: '>=8.6'} - picomatch@4.0.4: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} @@ -2710,8 +2673,8 @@ packages: peerDependencies: postcss: 8.5.28 - postcss-selector-parser@7.1.4: - resolution: {integrity: sha512-HeP7D2wyhkR+XaK6v4W8oRF62Dsz4flyuczALJp61GckGm42u1saSSJ/0auvcBqxs3jMRFEcPK34At/0JBKdOg==} + postcss-selector-parser@7.1.6: + resolution: {integrity: sha512-7qASPzhKF2l2KLboRZux8CCTRMdGiV08vWmyKzPz22qZ7ZjQBOeY7rNzNoCLSUiftJ7HUq0GERHmxw/t0dCdMw==} engines: {node: '>=4'} postcss@8.5.28: @@ -2817,10 +2780,6 @@ packages: resolution: {integrity: sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==} engines: {node: '>=0.12'} - readdirp@3.6.0: - resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} - engines: {node: '>=8.10.0'} - readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -2960,8 +2919,8 @@ packages: resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==} engines: {node: '>=18'} - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + source-map-js@1.2.2: + resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} engines: {node: '>=0.10.0'} space-separated-tokens@2.0.2: @@ -3104,10 +3063,6 @@ packages: resolution: {integrity: sha512-JIXCerhudr/N6OWLwLF1HVsTTUo7ry6qHa5eWZEkiMuxsIiAACL55tGLfqfHfoH7QaMQUW8fngD7u7TxWexYQg==} hasBin: true - to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} - totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -4056,7 +4011,7 @@ snapshots: jiti: 2.7.0 lightningcss: 1.32.0 magic-string: 0.30.21 - source-map-js: 1.2.1 + source-map-js: 1.2.2 tailwindcss: 4.3.3 '@tailwindcss/oxide-android-arm64@4.3.3': @@ -4446,11 +4401,6 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 - anymatch@3.1.3: - dependencies: - normalize-path: 3.0.0 - picomatch: 2.3.2 - arctic@3.7.0: dependencies: '@oslojs/crypto': 1.0.1 @@ -4487,8 +4437,6 @@ snapshots: dependencies: require-from-string: 2.0.2 - binary-extensions@2.3.0: {} - bits-ui@2.18.1(@internationalized/date@3.12.2)(@sveltejs/kit@2.70.3(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.56.4(@typescript-eslint/types@8.62.0))(vite@7.3.5(@types/node@24.13.4)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)))(svelte@5.56.4(@typescript-eslint/types@8.62.0))(typescript@5.9.3)(vite@7.3.5(@types/node@24.13.4)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)))(svelte@5.56.4(@typescript-eslint/types@8.62.0)): dependencies: '@floating-ui/core': 1.7.5 @@ -4506,10 +4454,6 @@ snapshots: dependencies: balanced-match: 4.0.4 - braces@3.0.3: - dependencies: - fill-range: 7.1.1 - call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -4525,18 +4469,6 @@ snapshots: character-entities@2.0.2: {} - chokidar@3.6.0: - dependencies: - anymatch: 3.1.3 - braces: 3.0.3 - glob-parent: 5.1.2 - is-binary-path: 2.1.0 - is-glob: 4.0.3 - normalize-path: 3.0.0 - readdirp: 3.6.0 - optionalDependencies: - fsevents: 2.3.3 - chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -4581,7 +4513,7 @@ snapshots: css-tree@3.2.1: dependencies: mdn-data: 2.27.1 - source-map-js: 1.2.1 + source-map-js: 1.2.2 cssesc@3.0.0: {} @@ -4978,10 +4910,6 @@ snapshots: dependencies: flat-cache: 4.0.1 - fill-range@7.1.1: - dependencies: - to-regex-range: 5.0.1 - find-up@5.0.0: dependencies: locate-path: 6.0.0 @@ -5030,10 +4958,6 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 - glob-parent@5.1.2: - dependencies: - is-glob: 4.0.3 - glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -5132,10 +5056,6 @@ snapshots: transitivePeerDependencies: - supports-color - is-binary-path@2.1.0: - dependencies: - binary-extensions: 2.3.0 - is-core-module@2.16.1: dependencies: hasown: 2.0.4 @@ -5148,8 +5068,6 @@ snapshots: is-module@1.0.0: {} - is-number@7.0.0: {} - is-plain-obj@4.1.0: {} is-potential-custom-element-name@1.0.1: {} @@ -5731,8 +5649,6 @@ snapshots: railroad-diagrams: 1.0.0 randexp: 0.4.6 - normalize-path@3.0.0: {} - obug@2.1.1: {} oniguruma-parser@0.12.2: {} @@ -5782,8 +5698,6 @@ snapshots: picocolors@1.1.1: {} - picomatch@2.3.2: {} - picomatch@4.0.4: {} playwright-core@1.61.1: {} @@ -5813,7 +5727,7 @@ snapshots: dependencies: postcss: 8.5.28 - postcss-selector-parser@7.1.4: + postcss-selector-parser@7.1.6: dependencies: cssesc: 3.0.0 util-deprecate: 1.0.2 @@ -5822,7 +5736,7 @@ snapshots: dependencies: nanoid: 3.3.19 picocolors: 1.1.1 - source-map-js: 1.2.1 + source-map-js: 1.2.2 prelude-ls@1.2.1: {} @@ -5863,10 +5777,6 @@ snapshots: discontinuous-range: 1.0.0 ret: 0.1.15 - readdirp@3.6.0: - dependencies: - picomatch: 2.3.2 - readdirp@4.1.2: {} redis-errors@1.2.0: {} @@ -6050,7 +5960,7 @@ snapshots: mrmime: 2.0.1 totalist: 3.0.1 - source-map-js@1.2.1: {} + source-map-js@1.2.2: {} space-separated-tokens@2.0.2: {} @@ -6098,7 +6008,7 @@ snapshots: espree: 10.4.0 postcss: 8.5.28 postcss-scss: 4.0.9(postcss@8.5.28) - postcss-selector-parser: 7.1.4 + postcss-selector-parser: 7.1.6 semver: 7.8.5 optionalDependencies: svelte: 5.56.4(@typescript-eslint/types@8.62.0) @@ -6209,10 +6119,6 @@ snapshots: dependencies: tldts-core: 7.0.29 - to-regex-range@5.0.1: - dependencies: - is-number: 7.0.0 - totalist@3.0.1: {} tough-cookie@6.0.1: @@ -6354,7 +6260,7 @@ snapshots: vite-plugin-static-copy@3.4.0(vite@7.3.5(@types/node@24.13.4)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)): dependencies: - chokidar: 3.6.0 + chokidar: 4.0.3 p-map: 7.0.4 picocolors: 1.1.1 tinyglobby: 0.2.16 diff --git a/src/lib/cluster-metrics-access.spec.ts b/src/lib/cluster-metrics-access.spec.ts new file mode 100644 index 00000000..2dc1ce41 --- /dev/null +++ b/src/lib/cluster-metrics-access.spec.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest'; + +import { clusterDashboardRedirect, withoutClusterMetricsNotice } from './cluster-metrics-access'; + +describe('clusterDashboardRedirect', () => { + const overview = '/c1/ws/dashboard/overview'; + + it('sends a denied viewer to the overview, which then says why', () => { + expect(clusterDashboardRedirect('denied', overview)).toBe(`${overview}?notice=cluster-metrics`); + }); + + it('keeps an allowed viewer, and one whose check failed', () => { + expect(clusterDashboardRedirect('allowed', overview)).toBeNull(); + expect(clusterDashboardRedirect('unknown', overview)).toBeNull(); + }); +}); + +describe('withoutClusterMetricsNotice', () => { + it('is null when there is no notice to show', () => { + expect(withoutClusterMetricsNotice(new URL('https://d/c1/ws/dashboard/overview'))).toBeNull(); + expect( + withoutClusterMetricsNotice(new URL('https://d/c1/ws/dashboard/overview?notice=other')) + ).toBeNull(); + }); + + it('drops only the notice, keeping the rest of the URL', () => { + const url = new URL('https://d/c1/ws/dashboard/overview?notice=cluster-metrics&tab=x#top'); + const cleaned = withoutClusterMetricsNotice(url); + expect(cleaned?.href).toBe('https://d/c1/ws/dashboard/overview?tab=x#top'); + expect(url.searchParams.get('notice')).toBe('cluster-metrics'); + }); +}); diff --git a/src/lib/cluster-metrics-access.ts b/src/lib/cluster-metrics-access.ts new file mode 100644 index 00000000..d34b5f16 --- /dev/null +++ b/src/lib/cluster-metrics-access.ts @@ -0,0 +1,33 @@ +/** + * Whether the viewer may read cluster-scope metrics on this cluster, as the + * (auth) layout learns it (src/lib/server/metrics-access.ts). + * + * unknown: the question could not be asked. Pages draw what a viewer without + * access sees — the server would answer the rest 403 or 503 anyway — but say + * so in a banner, and do not move the viewer elsewhere. + */ +export type ClusterMetricsAccess = 'allowed' | 'denied' | 'unknown'; + +const NOTICE_PARAM = 'notice'; +const CLUSTER_METRICS_NOTICE = 'cluster-metrics'; + +/** + * Where a cluster-wide dashboard (Cluster, Storage) sends the viewer, or null + * to let them stay. Only a denial leaves: on unknown it was the check that + * failed, not the viewer, and a redirect would only lose an admin's bookmark. + * The overview reads the notice and says why the viewer landed there. + */ +export function clusterDashboardRedirect( + access: ClusterMetricsAccess, + overviewPath: string +): string | null { + return access === 'denied' ? `${overviewPath}?${NOTICE_PARAM}=${CLUSTER_METRICS_NOTICE}` : null; +} + +/** `url` without the redirect notice, or null when it carries none. `url` is not changed. */ +export function withoutClusterMetricsNotice(url: URL): URL | null { + if (url.searchParams.get(NOTICE_PARAM) !== CLUSTER_METRICS_NOTICE) return null; + const cleaned = new URL(url); + cleaned.searchParams.delete(NOTICE_PARAM); + return cleaned; +} diff --git a/src/lib/components/dashboard/cluster/index.svelte b/src/lib/components/dashboard/cluster/index.svelte index 76b603bb..fa35bfb4 100644 --- a/src/lib/components/dashboard/cluster/index.svelte +++ b/src/lib/components/dashboard/cluster/index.svelte @@ -1,6 +1,6 @@ @@ -256,15 +280,19 @@
-

- {restartTotal} +

+ {mainUnavailable ? '—' : restartTotal}

{m.metric_restart()}

- {#if restartByPod.length > 0} + {#if mainUnavailable} + +

{m.pod_health_unavailable()}

+ + {:else if restartByPod.length > 0}
    {#each restartByPod as p (p.pod)} @@ -281,15 +309,19 @@
    -

    - {oomTotal} +

    + {mainUnavailable ? '—' : oomTotal}

    {m.metric_oom()}

    - {#if oomByPod.length > 0} + {#if mainUnavailable} + +

    {m.pod_health_unavailable()}

    +     + {:else if oomByPod.length > 0}
      {#each oomByPod as p (p.pod)} @@ -306,15 +338,24 @@
      -

      - {nodeCount === 0 ? '—' : xidTotal} +

      + {nodeCount === 0 || xidUnavailable ? '—' : xidTotal}

      {m.metric_xid()}

      - {#if nodeCount === 0} + {#if xidUnavailable} + +

      {m.xid_unavailable()}

      +       + {:else if nodeCount === 0}

      {m.no_gpu_for_model()}

       @@ -336,8 +377,8 @@
      -

      - {throttlePct.toFixed(1)}% +

      + {mainUnavailable ? '—' : `${throttlePct.toFixed(1)}%`}

      {m.metric_cpu_throttle()} @@ -345,15 +386,17 @@

      -

      {m.llm_dashboard_cpu_throttle_hint()}

      +

      + {mainUnavailable ? m.pod_health_unavailable() : m.llm_dashboard_cpu_throttle_hint()} +

      -

      - {shmCrashCount} +

      + {mainUnavailable ? '—' : shmCrashCount}

      {m.metric_shm_crash()} @@ -361,7 +404,9 @@

      - {#if shmCrashPods.length === 0} + {#if mainUnavailable} +

      {m.pod_health_unavailable()}

      + {:else if shmCrashPods.length === 0}

      {m.llm_dashboard_shm_crash_hint()}

      {:else}
      @@ -384,8 +429,8 @@
      -

      - {preemptTotal} +

      + {mainUnavailable ? '—' : preemptTotal}

      {m.metric_preempt()} @@ -393,7 +438,9 @@

      - {#if preemptByPod.length === 0} + {#if mainUnavailable} +

      {m.pod_health_unavailable()}

      + {:else if preemptByPod.length === 0}

      {m.llm_dashboard_preempt_hint()}

      {:else}
      diff --git a/src/lib/components/dashboard/model/analytics/index.svelte b/src/lib/components/dashboard/model/analytics/index.svelte index 88b9dad1..9a3f8da1 100644 --- a/src/lib/components/dashboard/model/analytics/index.svelte +++ b/src/lib/components/dashboard/model/analytics/index.svelte @@ -2,6 +2,7 @@ import { PrometheusDriver } from 'prometheus-query'; import { m } from '$lib/messages'; + import type { SharedMetrics } from '$lib/prometheus'; import ATopKvPressure from './a-top-kv-pressure.svelte'; import ATopP99Latency from './a-top-p99-latency.svelte'; @@ -31,6 +32,7 @@ start, end, endIsNow, + shared, isReloading = $bindable() }: { cluster: string; @@ -40,6 +42,8 @@ start: Date; end: Date; endIsNow: boolean; + /** The shared scope the GPU panels ask, as the Workspace's member. */ + shared: SharedMetrics; isReloading?: boolean; } = $props(); @@ -105,6 +109,7 @@ prometheusDriver={client} selectedModel={modelFilter} isReloading={isReloading ?? false} + {shared} /> import LoaderCircle from '@lucide/svelte/icons/loader-circle'; - import { PrometheusDriver } from 'prometheus-query'; + import type { PrometheusDriver } from 'prometheus-query'; import { onDestroy, onMount } from 'svelte'; import { @@ -18,12 +18,14 @@ import * as Item from '$lib/components/ui/item'; import * as Tabs from '$lib/components/ui/tabs'; import { m } from '$lib/messages'; + import { dashboardBody, sharedMetrics, workspacePrometheusDriver } from '$lib/prometheus'; let { cluster, currentWorkspace = '', defaultNamespace = '', - isClusterAdmin = false + isClusterAdmin = false, + canViewClusterMetrics = false }: { cluster: string; /** Name of the currently active workspace (for default picker selection) */ @@ -31,10 +33,23 @@ /** Workspace namespace used as initial namespace state */ defaultNamespace?: string; isClusterAdmin?: boolean; + canViewClusterMetrics?: boolean; } = $props(); let isReloading = $state(true); let prometheusDriver = $state(null); + // GPU panels ask the shared scope as the Workspace's member, whichever + // namespace an admin picks for the charts. + const shared = $derived(sharedMetrics(cluster, defaultNamespace)); + let initialized = $state(false); + const body = $derived( + dashboardBody({ + hasDriver: prometheusDriver !== null, + initialized, + cluster, + namespace: defaultNamespace + }) + ); let selectedTab = $state('overview'); /** Currently selected namespace; empty string means "all namespaces" */ let selectedNamespace = $state(getInitialNamespace()); @@ -60,16 +75,15 @@ onMount(async () => { try { - prometheusDriver = new PrometheusDriver({ - endpoint: `/proxy/${cluster}/prometheus`, - baseURL: '/api/v1', - headers: { - 'x-proxy-target': 'api' - } - }); + prometheusDriver = workspacePrometheusDriver( + cluster, + defaultNamespace, + canViewClusterMetrics + ); } catch (error) { console.error('Failed to initialize Prometheus driver:', error); } + initialized = true; }); onDestroy(() => { @@ -113,12 +127,16 @@ bind:selectedModel />
      - + + {#if isClusterAdmin && canViewClusterMetrics} + + {/if} {/key} @@ -160,10 +179,14 @@
      - {:else if cluster} + {:else if body === 'loading'}
      + {:else if body === 'no-namespace'} +
      +

      {m.workspace_namespace_unknown()}

      +
      {:else}

      {m.no_data_display()}

      diff --git a/src/lib/components/dashboard/namespace-scope.spec.ts b/src/lib/components/dashboard/namespace-scope.spec.ts new file mode 100644 index 00000000..0507d4a3 --- /dev/null +++ b/src/lib/components/dashboard/namespace-scope.spec.ts @@ -0,0 +1,141 @@ +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +function files(dir: string): string[] { + return readdirSync(dir).flatMap((name) => { + const path = join(dir, name); + return statSync(path).isDirectory() ? files(path) : [path]; + }); +} + +// Namespace scope forces namespace="". KubeVirt's PodMonitor keeps the +// VMI's own namespace label (helm-charts kubevirt 1.2.0), so a selector on +// exported_namespace would only ever match nothing for a tenant. +describe('compute dashboard', () => { + it('selects VM metrics on namespace, not exported_namespace', () => { + const dir = join(process.cwd(), 'src/lib/components/dashboard/compute'); + const offenders = files(dir).filter((f) => + readFileSync(f, 'utf8').includes('exported_namespace') + ); + expect(offenders).toEqual([]); + }); + + it('builds namespace matchers with namespaceSelector, so "all namespaces" selects all', () => { + const dir = join(process.cwd(), 'src/lib/components/dashboard/compute'); + const offenders = files(dir).filter((f) => /namespace="\$\{/.test(readFileSync(f, 'utf8'))); + expect(offenders).toEqual([]); + }); +}); + +/** + * Everything a Workspace page renders: its routes and the components they use. + * The cluster and storage dashboards live under the Workspace route too, but + * only viewers with cluster metrics access reach them. + */ +const workspaceRoots = [ + 'src/lib/components/dashboard/workspace', + 'src/lib/components/dashboard/model', + 'src/lib/components/dashboard/compute', + 'src/lib/components/gpu-allocation', + 'src/lib/components/kind-viewer', + 'src/routes/(auth)/[cluster]/[workspace]' +]; +const clusterOnly = [ + 'src/lib/components/dashboard/cluster/', + 'src/lib/components/dashboard/storage/', + 'src/routes/(auth)/[cluster]/[workspace]/dashboard/cluster/', + 'src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/' +]; + +function workspaceFiles(): string[] { + return workspaceRoots + .flatMap((root) => files(join(process.cwd(), root))) + .map((f) => relative(process.cwd(), f).split(sep).join('/')) + .filter((f) => /\.(svelte|ts)$/.test(f) && !f.endsWith('.spec.ts')) + .filter((f) => !clusterOnly.some((dir) => f.startsWith(dir))); +} + +/** + * $lib/prometheus exports that reach past the Workspace's namespace: the + * cluster-scope driver, and PromQL pieces over node, GPU and disk exporters. + */ +const infrastructureHelpers = [ + 'clusterPrometheusDriver', + 'dcgmNodeSelector', + 'DCGM_GPU_MEMORY_TOTAL_BYTES', + 'DCGM_GPU_MEMORY_USED_BYTES', + 'ai100DiskRate', + 'ai100DiskRateByNode', + 'AIDAPTIV_CACHE_DEVICE_CLASS' +]; + +/** Infrastructure metrics: their series can name other tenants' pods, or none. */ +const infrastructureMetrics = + /DCGM_FI_|hami_|kubevirt_info|kubevirt_virt_controller|\bceph_|node_disk_|node_uname_info|topolvm_/; + +const forbidden: [string, RegExp][] = [ + ['new PrometheusDriver', /\bnew\s+PrometheusDriver\b/], + ...infrastructureHelpers.map((name): [string, RegExp] => [name, new RegExp(`\\b${name}\\b`)]), + ['infrastructure metric', infrastructureMetrics] +]; + +// Workspace pages make drivers with workspacePrometheusDriver, which keeps a +// viewer without cluster metrics access at namespace scope, and take GPU, +// KubeVirt and Ceph numbers from the server's named queries (shared scope), +// for admins and members alike. No file they render may pick the cluster +// scope or carry infrastructure PromQL of its own. +describe('workspace pages', () => { + const scanned = workspaceFiles(); + + it('scans every directory a Workspace page renders from', () => { + for (const root of workspaceRoots) expect(existsSync(join(process.cwd(), root))).toBe(true); + expect(scanned).toContain('src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte'); + expect(scanned).toContain( + 'src/routes/(auth)/[cluster]/[workspace]/dashboard/model/+page.svelte' + ); + expect(scanned.some((f) => clusterOnly.some((dir) => f.startsWith(dir)))).toBe(false); + }); + + it('forbids helpers that $lib/prometheus still exports', () => { + const source = readFileSync(join(process.cwd(), 'src/lib/prometheus.ts'), 'utf8'); + for (const name of infrastructureHelpers) { + expect(source).toMatch(new RegExp(`export (const|function) ${name}\\b`)); + } + }); + + it('use no cluster-scope driver and no infrastructure PromQL', () => { + const offenders = scanned.flatMap((file) => { + const source = readFileSync(join(process.cwd(), file), 'utf8'); + return forbidden + .filter(([, pattern]) => pattern.test(source)) + .map(([name]) => `${file}: ${name}`); + }); + expect(offenders).toEqual([]); + }); +}); + +// The places that show cluster-wide numbers on a Workspace page, and the +// shared-scope calls they make for them. +describe('shared-scope components', () => { + const sharedFiles = [ + 'src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte', + 'src/lib/components/gpu-allocation/fetch-utilization.ts', + 'src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte', + 'src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte', + 'src/lib/components/dashboard/model/analytics/b-pod-health.svelte', + 'src/lib/components/dashboard/compute/overview/controller.svelte', + 'src/lib/components/dashboard/compute/overview/worker.svelte', + 'src/lib/components/dashboard/workspace/overview/storage.svelte' + ]; + const sharedCall = + /\bshared\.(instant|range)\(\s*'[a-z-]+'|\bfetchGpu(Utilization|AllocatedTotals)\(\s*shared\b/; + + for (const file of sharedFiles) { + it(`${file} queries the shared scope`, () => { + const source = readFileSync(join(process.cwd(), file), 'utf8'); + expect(source).toMatch(sharedCall); + }); + } +}); diff --git a/src/lib/components/dashboard/storage/index.svelte b/src/lib/components/dashboard/storage/index.svelte index cb51daea..fe80b629 100644 --- a/src/lib/components/dashboard/storage/index.svelte +++ b/src/lib/components/dashboard/storage/index.svelte @@ -1,5 +1,5 @@ - - - {#if trigger} - {@render trigger()} - {:else} - - - - - - {m.gpu_allocation()} - - - {/if} - - - - - {m.gpu_allocation_title({ name })} - - - {#if view === 'llm-inference-service'} - {m.gpu_allocation_model_description()} - {:else} - {m.gpu_allocation_node_description()} - {/if} - - - -
      - {#if isLoading} -
      - {m.gpu_allocation_loading()} -
      - {:else if error} -
      - {error} -
      - {:else if isEmpty} -
      +{#if available} + + + {#if trigger} + {@render trigger()} + {:else} + + + + + + {m.gpu_allocation()} + + + {/if} + + + + + {m.gpu_allocation_title({ name })} + + {#if view === 'llm-inference-service'} - {m.gpu_allocation_empty_model()} + {m.gpu_allocation_model_description()} {:else} - {m.gpu_allocation_empty_node()} + {m.gpu_allocation_node_description()} {/if} -
      - {:else if topologyData} - - {/if} -
      -       -       + + + +
      + {#if isLoading} +
      + {m.gpu_allocation_loading()} +
      + {:else if error} +
      + {error} +
      + {:else if isEmpty} +
      + {#if view === 'llm-inference-service'} + {m.gpu_allocation_empty_model()} + {:else} + {m.gpu_allocation_empty_node()} + {/if} +
      + {:else if topologyData} + + {/if} +
      + + +{/if} diff --git a/src/lib/components/gpu-allocation/types.ts b/src/lib/components/gpu-allocation/types.ts index 54711627..6efd06be 100644 --- a/src/lib/components/gpu-allocation/types.ts +++ b/src/lib/components/gpu-allocation/types.ts @@ -58,12 +58,24 @@ export interface GpuUtilization { totalMem?: number; } +/** + * What hami-scheduler has booked on a card across every workload: memory in MiB + * (the unit GpuDevice speaks) and how many containers share it. Names no pod — + * it is what a viewer who may not list other tenants' pods sees instead. + */ +export interface GpuAllocatedTotal { + usedMem: number; + shareCount: number; +} + export interface GpuInfo { device: GpuDevice; nodeName: string; allocatedBy: { podName: string; podNamespace: string; usedCores: number; usedMem: number }[]; /** Measured usage of the card; absent when DCGM reports nothing for it. */ utilization?: GpuUtilization; + /** Per-card totals for viewers who may not list every pod on the node. */ + allocatedTotal?: GpuAllocatedTotal; } export interface NodeInfo { diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte index 93e776e2..02f38d72 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte @@ -2,6 +2,7 @@ import Ellipsis from '@lucide/svelte/icons/ellipsis'; import type { Schema } from '@sjsf/form'; + import { page } from '$app/state'; import Describe from '$lib/components/kind-viewer/kind-viewer-actions/default/describe.svelte'; import View from '$lib/components/kind-viewer/kind-viewer-actions/default/view.svelte'; import Button from '$lib/components/ui/button/button.svelte'; @@ -46,19 +47,22 @@ - { - e.preventDefault(); - }} - > - { - actionsOpen = false; + + {#if page.data.canViewClusterMetrics === true} + { + e.preventDefault(); }} - /> - + > + { + actionsOpen = false; + }} + /> + + {/if} { e.preventDefault(); diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte index e00949ff..bab55e85 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte @@ -2,7 +2,6 @@ import type { JsonValue } from '@bufbuild/protobuf'; import { GpuIcon } from '@lucide/svelte'; import lodash from 'lodash'; - import { PrometheusDriver } from 'prometheus-query'; import { onMount } from 'svelte'; import { page } from '$app/state'; @@ -11,68 +10,78 @@ import * as Modal from '$lib/components/ui/dialog'; import * as Item from '$lib/components/ui/item'; import * as Table from '$lib/components/ui/table'; + import { m } from '$lib/messages'; + import { sharedMetrics } from '$lib/prometheus'; let gpus: Record[] = []; let gpusByModelName: Record[] = []; let gpusByHostName: Record[] = []; - - const driver = new PrometheusDriver({ - endpoint: `/proxy/${page.params.cluster!}/prometheus`, - baseURL: '/api/v1', - headers: { 'x-proxy-target': 'api' } - }); + /** Set when the numbers could not be loaded, so an empty table is not read as no GPUs. */ + let errorMessage: string | null = null; onMount(async () => { - const [fieldBufferResponse, vgpuLimitResponse, vgpuUsageResponse] = await Promise.all([ - driver.instantQuery( - '(DCGM_FI_DEV_FB_FREE + DCGM_FI_DEV_FB_RESERVED + DCGM_FI_DEV_FB_USED) * (1024 * 1024)' // DCGM return in Mi - ), - driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_limit_bytes)'), - driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_used_bytes)') - ]); + // Per-card inventory and vGPU totals: cluster-wide, naming no workload, + // from the shared scope for every viewer. + const shared = sharedMetrics(page.params.cluster!, page.data.namespace ?? ''); - const vgpuLimits = vgpuLimitResponse.result.map((series) => { - return { ...series.metric.labels, limit: series.value?.value }; - }); - const vgpuUsages = vgpuUsageResponse.result.map((series) => { - return { ...series.metric.labels, usage: series.value?.value }; - }); + try { + const [fieldBufferResponse, vgpuLimitResponse, vgpuUsageResponse] = await Promise.all([ + shared.instant('gpu-inventory'), + shared.instant('vgpu-memory-limit'), + shared.instant('vgpu-memory-used') + ]); - const limitsByDeviceUUID = lodash.mapValues( - lodash.groupBy(vgpuLimits, 'device_uuid'), - (group) => lodash.sumBy(group, 'limit') - ); - const usagesByDeviceUUID = lodash.mapValues( - lodash.groupBy(vgpuUsages, 'device_uuid'), - (group) => lodash.sumBy(group, 'usage') - ); + const vgpuLimits = vgpuLimitResponse.result.map((series) => { + return { ...series.metric.labels, limit: series.value?.value }; + }); + const vgpuUsages = vgpuUsageResponse.result.map((series) => { + return { ...series.metric.labels, usage: series.value?.value }; + }); - gpus = fieldBufferResponse.result.map((series) => { - const labels = series.metric.labels; - const uuid = lodash.get(labels, 'UUID'); - return { - ...labels, - total: lodash.get(series, ['value', 'value'], undefined), - limit: lodash.get(limitsByDeviceUUID, uuid, undefined), - usage: lodash.get(usagesByDeviceUUID, uuid, undefined) - }; - }); - gpusByModelName = Object.entries( - lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'modelName')) - ).map(([key, value]) => ({ - identifier: key, - limit: lodash.sumBy(value, 'limit'), - usage: lodash.sumBy(value, 'usage'), - total: lodash.sumBy(value, 'total') - })); - gpusByHostName = Object.entries(lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'hostname'))).map( - ([key, value]) => ({ + const limitsByDeviceUUID = lodash.mapValues( + lodash.groupBy(vgpuLimits, 'device_uuid'), + (group) => lodash.sumBy(group, 'limit') + ); + const usagesByDeviceUUID = lodash.mapValues( + lodash.groupBy(vgpuUsages, 'device_uuid'), + (group) => lodash.sumBy(group, 'usage') + ); + + gpus = fieldBufferResponse.result.map((series) => { + const labels = series.metric.labels; + const uuid = lodash.get(labels, 'UUID'); + return { + ...labels, + total: lodash.get(series, ['value', 'value'], undefined), + limit: lodash.get(limitsByDeviceUUID, uuid, undefined), + usage: lodash.get(usagesByDeviceUUID, uuid, undefined) + }; + }); + gpusByModelName = Object.entries( + lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'modelName')) + ).map(([key, value]) => ({ + identifier: key, + limit: lodash.sumBy(value, 'limit'), + usage: lodash.sumBy(value, 'usage'), + total: lodash.sumBy(value, 'total') + })); + gpusByHostName = Object.entries( + lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'hostname')) + ).map(([key, value]) => ({ identifier: key, limit: lodash.sumBy(value, 'limit'), usage: lodash.sumBy(value, 'usage'), total: lodash.sumBy(value, 'total') - }) - ); + })); + } catch (error) { + console.warn('Failed to fetch GPU resources:', error); + gpus = []; + gpusByModelName = []; + gpusByHostName = []; + errorMessage = m.gpu_resources_fetch_error({ + message: error instanceof Error ? error.message : String(error) + }); + } }); @@ -89,172 +98,178 @@ GPU Resources Amount of limits, usage, and total field buffers. - {gpus.length} devices + {#if !errorMessage} + {gpus.length} devices + {/if} - - - Summary - - {gpusByModelName.length} types - - - - - - - Identifier - usage - limit - total - - - - {#each gpusByModelName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + {#if errorMessage} +

      {errorMessage}

      + {:else} + + + Summary + + {gpusByModelName.length} types + + + + + + + Identifier + usage + limit + total - {/each} -       -       + + + {#each gpusByModelName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + - - - Summary - - {gpusByHostName.length} hosts - - - - - - - Identifier - usage - limit - total - - - - {#each gpusByHostName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + + + Summary + + {gpusByHostName.length} hosts + + + + + + + Identifier + usage + limit + total - {/each} - - + + + {#each gpusByHostName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + - - - Details - - {gpus.length} devices - - - - - - - Model Name - Host Name - Device - UUID - usage - limit - total - - - - {#each gpus as gpu, index (index)} - - {gpu.modelName} - {gpu.hostname} - {gpu.device} - {gpu.UUID} - - {@const { value, unit } = - gpu.usage && typeof gpu.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpu.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.limit && typeof gpu.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpu.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.total && typeof gpu.total === 'number' - ? formatWithBinarySuffix(BigInt(gpu.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + + + Details + + {gpus.length} devices + + + + + + + Model Name + Host Name + Device + UUID + usage + limit + total - {/each} - - + + + {#each gpus as gpu, index (index)} + + {gpu.modelName} + {gpu.hostname} + {gpu.device} + {gpu.UUID} + + {@const { value, unit } = + gpu.usage && typeof gpu.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpu.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.limit && typeof gpu.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpu.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.total && typeof gpu.total === 'number' + ? formatWithBinarySuffix(BigInt(gpu.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + + {/if} diff --git a/src/lib/components/resource-viewer/viewers/workspaces-viewer.svelte b/src/lib/components/resource-viewer/viewers/workspaces-viewer.svelte deleted file mode 100644 index 7fd3ccab..00000000 --- a/src/lib/components/resource-viewer/viewers/workspaces-viewer.svelte +++ /dev/null @@ -1,834 +0,0 @@ - - -{#if !isLoaded} - - - {#each Array(13).keys() as index (index)} - {#if index % 2 === 0} - {#if index % 3 !== 0} - {#if index % 5 === 0} - {#if index % 7 !== 0} - {#if index % 11 === 0} - - {:else} - - {/if} - {:else} - - {/if} - {:else} - - {/if} - {:else} - - {/if} - {:else} - - {/if} - {/each} - - -{:else} - {#snippet memberGrid()} -
      - {#each filteredMembers as member (member.subject)} - - - - {getMemberInitials(member)} - - - - - {getMemberLabel(member)} - - {#if member.serviceAccount} - service account - {/if} - {member.role} - - - - - - {#if member.username} - {member.username} - {/if} - {member.subject} - - - {#if member.username} -

      username: {member.username}

      - {/if} -

      subject: {member.subject}

      -       -       -       -       -       - {/each} -
      - {/snippet} - - - - - - {@const conditions = object.status?.conditions ?? []} - {@const readyCondition = conditions.find((condition) => condition.type === 'Ready')} - {@const isReady = readyCondition?.status === 'True' ? true : false} - - - - - - - - Status - - - - - - {#if conditions.length > 0} - {#if isReady} - - - - {readyCondition?.reason} - - {readyCondition?.message} - - - {:else} - {#each [...conditions].sort((p, n) => { - const previous = new Date(p.lastTransitionTime ?? 0).getTime(); - const next = new Date(n.lastTransitionTime ?? 0).getTime(); - return next - previous; - }) as condition, index (index)} - - - - {condition?.reason} - - {condition?.message} - - - {/each} - {/if} - {:else} - - - - - - No Status Available - - There are no status conditions to display for this workspace. - - - - {/if} - - - - - - {@const resourceQuotaHard = object.spec?.resourceQuota?.hard ?? {}} - - - - - - - - Resource Quota - - - - - {#snippet child({ props })} - { - isResourceQuotasGrid = !isResourceQuotasGrid; - }} - class="data-[state=on]:*:[svg]:fill-muted-foreground/50" - > - - - {/snippet} - - Toggle Grid Layout - - - - - - - {#if Object.keys(resourceQuotaHard).length > 0} -
      - {#each Object.keys(resourceQuotaHard) as key, index (index)} - - - - {key} - - - {resourceQuotaUsed[key] !== undefined - ? resourceQuotaUsed[key] - : '?'}/{formatHardValue(key, resourceQuotaHard[key])} - - - - {/each} -
      - {:else} - - - - - - No Resource Quota Configured - - Resource Quota is not configured yet. Please click the edit button at the top - right to configure Resource Quota. - - - - {/if} -       -       - - - - {@const limits = object.spec?.limitRange?.limits ?? []} - - - - - - - - Limit Range - - - - - - {#if limits.length > 0} - {#each limits as limit, index (index)} - {@const { type, ...thresholds } = limit} - - - - {type} - - - - {#each Object.entries(thresholds) as [key, values], index (index)} - {#if values && typeof values === 'object'} - - - - {key} - - - {#each Object.entries(values) as [key, value], index (index)} -

      {key}:{value}

      - {/each} -       -       -       - {/if} - {/each} -       -       - - {/each} - {:else} - - - - - - No Limit Range Configured - - Limit Range is not configured yet. Please click the edit button at the top right - to configure Limit Range. - - - - {/if} -       -       - - - - - - - - - - - - Network Isolation - - - - - - - - - Enabled - - {@const enabled = object.spec?.networkIsolation?.enabled ?? null} - {#if enabled === true} - - {:else} - - {/if} - - - - - {@const allowedNamespaces = object.spec?.networkIsolation?.allowedNamespaces ?? []} - - - - Allowed Namespaces - - - {#if allowedNamespaces.length > 0} -
      - {#each allowedNamespaces as allowedNamespace, index (index)} - - - {allowedNamespace} - - {/each} -
      - {:else} - - -

      No namespaces allowed

      -       - {/if} -       -       - - {allowedNamespaces.length} - -       -       -       - - - - - - - - - - - - Members - {#if memberRoleSummary} - {memberRoleSummary} - {/if} - - - - -
      - {#if members.length > MEMBERS_SCROLL_THRESHOLD} - - - - - {#snippet child({ props })} - - - - {/snippet} - - - - Filter by Role - - {#each MEMBER_ROLE_FILTERS as roleFilter (roleFilter)} - - {roleFilter} - - {roleFilter === 'all' ? members.length : memberRoleCounts[roleFilter]} - - - {/each} - - - - Filter by Role - - {/if} - - - {filteredMembers.length} - -
      -       -       - - {#if members.length === 0} - - - - - - No Members - - No members have been granted access to this workspace yet. - - - - {:else if filteredMembers.length === 0} - - - - - - No Matching Members - - No member holds the {memberRoleFilter} role in this workspace. - - - - {:else if filteredMembers.length > MEMBERS_SCROLL_THRESHOLD} - -
      - {@render memberGrid()} -
      -       - {:else} - {@render memberGrid()} - {/if} -       -       -       - - {@const namespace = object.status?.namespaceRef?.name ?? ''} - - {#if relatedResources.length > 0} - - - - Related Resources - - {relatedResources.length} resources related to {object.kind} - {object.metadata?.name} - - - - -
      - {#each relatedResources as relatedResource (relatedResource.kind + relatedResource.name)} - {#if !relatedResource.namespaced || (relatedResource.namespaced && namespace)} - - {#snippet child({ props })} - {@const urlSearchParameters = new URLSearchParams({ - group: relatedResource.group, - version: relatedResource.version, - kind: relatedResource.kind, - resource: relatedResource.resource, - ...(relatedResource.namespaced ? { namespace: namespace } : {}) - })} - {@const url = resolve( - `/(auth)/${page.params.cluster}/${page.params.workspace}/${relatedResource.name}?${urlSearchParameters}` - )} - - - {relatedResource.name} - - {relatedResource.resource}.{relatedResource.group - ? `${relatedResource.group}/${relatedResource.version}` - : relatedResource.version} - - - - - - - {/snippet} - - {/if} - {/each} -
      - {:else} - - - - - - No Related Resources - - There are no related resources to display for this workspace. - - - - {/if} -       -       -{/if} diff --git a/src/lib/prometheus.spec.ts b/src/lib/prometheus.spec.ts index c1018062..0f91bfe5 100644 --- a/src/lib/prometheus.spec.ts +++ b/src/lib/prometheus.spec.ts @@ -1,6 +1,22 @@ -import { describe, expect, it } from 'vitest'; +import { createServer, type IncomingMessage, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; -import { classifyGpuGovernance } from './prometheus'; +import type { PrometheusDriver } from 'prometheus-query'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +import { + classifyGpuGovernance, + clusterPrometheusDriver, + dashboardBody, + fetchCombinedInstant, + fetchCombinedInstantOrNull, + fetchModelNodes, + namespacePrometheusDriver, + namespaceSelector, + sharedMetrics, + type SharedQueryName, + workspacePrometheusDriver +} from './prometheus'; const GIB = 1024 * 1024 * 1024; const MIB = 1024 * 1024; @@ -47,3 +63,199 @@ describe('classifyGpuGovernance', () => { expect(governance).toHaveProperty('bytes', 8_388_608_000 - 1_048_576); }); }); + +describe('prometheus drivers', () => { + let server: Server; + let origin = ''; + const seen: URL[] = []; + + beforeAll(async () => { + server = createServer((req: IncomingMessage, res) => { + const url = new URL(req.url ?? '/', 'http://x'); + seen.push(url); + if (url.pathname.endsWith('/prometheus-shared/broken')) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + status: 'error', + errorType: 'unavailable', + error: 'cannot check metrics access on this cluster; retry shortly' + }) + ); + return; + } + // series and label endpoints answer with a bare array, query endpoints with a result. + const data = /\/(series|label\/)/.test(url.pathname) + ? [] + : { resultType: 'vector', result: [] }; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ status: 'success', data })); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(() => new Promise((resolve) => server.close(() => resolve()))); + + it('namespace scope adds namespace to every request', async () => { + seen.length = 0; + const driver = namespacePrometheusDriver('c1', 'ws-a', origin); + await driver.instantQuery('up'); + await driver.rangeQuery('up', new Date(0), new Date(60_000), 15); + await driver.series('up', new Date(0), new Date(60_000)); + await driver.labelValues('pod'); + + expect(seen.map((u) => u.pathname)).toEqual([ + '/proxy/c1/prometheus-tenancy/api/v1/query', + '/proxy/c1/prometheus-tenancy/api/v1/query_range', + '/proxy/c1/prometheus-tenancy/api/v1/series', + '/proxy/c1/prometheus-tenancy/api/v1/label/pod/values' + ]); + for (const u of seen) expect(u.searchParams.getAll('namespace')).toEqual(['ws-a']); + expect(seen[0].searchParams.get('query')).toBe('up'); + }); + + it('cluster scope sends no namespace', async () => { + seen.length = 0; + await clusterPrometheusDriver('c1', origin).instantQuery('up'); + expect(seen[0].pathname).toBe('/proxy/c1/prometheus/api/v1/query'); + expect(seen[0].searchParams.has('namespace')).toBe(false); + }); + + it('workspace pages pick the scope by access', () => { + expect(workspacePrometheusDriver('c1', 'ws-a', false)).not.toBeNull(); + expect(workspacePrometheusDriver('c1', '', false)).toBeNull(); + expect(workspacePrometheusDriver('c1', '', true)).not.toBeNull(); + }); + + it('shared scope sends a name and parameters, never PromQL', async () => { + seen.length = 0; + const shared = sharedMetrics('c1', 'ws-a', origin); + const result = await shared.instant( + 'gpu-util', + { uuid: ['GPU-a', 'GPU-b'] }, + new Date(1_700_000_000_000) + ); + await shared.range( + 'host-gpu-fb-used', + { hostname: ['hci'] }, + new Date(0), + new Date(60_000), + 120 + ); + + expect(result.resultType).toBe('vector'); + expect(seen.map((u) => u.pathname)).toEqual([ + '/proxy/c1/prometheus-shared/gpu-util', + '/proxy/c1/prometheus-shared/host-gpu-fb-used' + ]); + expect(seen[0].searchParams.getAll('uuid')).toEqual(['GPU-a', 'GPU-b']); + expect(seen[0].searchParams.get('time')).toBe('1700000000'); + expect(seen[1].searchParams.getAll('hostname')).toEqual(['hci']); + expect(seen[1].searchParams.get('start')).toBe('0'); + expect(seen[1].searchParams.get('end')).toBe('60'); + expect(seen[1].searchParams.get('step')).toBe('120'); + for (const u of seen) { + expect(u.searchParams.getAll('namespace')).toEqual(['ws-a']); + expect(u.searchParams.has('query')).toBe(false); + } + }); + + it('shared scope leaves namespace out when the Workspace has none', async () => { + seen.length = 0; + await sharedMetrics('c1', '', origin).instant('kubevirt-handlers'); + expect([...seen[0].searchParams.keys()]).toEqual([]); + }); + + it("shared scope rejects with the server's message", async () => { + await expect( + sharedMetrics('c1', 'ws-a', origin).instant('broken' as SharedQueryName) + ).rejects.toThrow('cannot check metrics access'); + }); +}); + +describe('dashboardBody', () => { + const base = { hasDriver: false, initialized: true, cluster: 'c1', namespace: 'ws-a' }; + + it('shows charts whenever there is a driver', () => { + // An admin keeps cluster scope even when the Workspace's namespace is unknown. + expect(dashboardBody({ ...base, hasDriver: true, namespace: '' })).toBe('charts'); + }); + + it('spins only until the driver has been made', () => { + expect(dashboardBody({ ...base, initialized: false })).toBe('loading'); + }); + + it('explains a missing namespace instead of spinning', () => { + expect(dashboardBody({ ...base, namespace: '' })).toBe('no-namespace'); + }); + + it('says there is no data otherwise', () => { + expect(dashboardBody({ ...base, cluster: '' })).toBe('empty'); + expect(dashboardBody(base)).toBe('empty'); + }); +}); + +describe('namespaceSelector', () => { + it('selects one namespace, escaped', () => { + expect(namespaceSelector('ws-a')).toBe('{namespace="ws-a"}'); + expect(namespaceSelector('a"b')).toBe('{namespace="a\\"b"}'); + }); + + it('selects every namespace for the empty choice', () => { + expect(namespaceSelector('')).toBe(''); + }); +}); + +describe('fetchModelNodes', () => { + const driver = (instantQuery: () => Promise) => + ({ instantQuery }) as unknown as PrometheusDriver; + + it('resolves null when the lookup fails, not "no nodes"', async () => { + const failing = driver(async () => { + throw new Error('cannot check metrics access on this cluster; retry shortly'); + }); + await expect(fetchModelNodes(failing, 'ws-a', 'm1')).resolves.toBeNull(); + }); + + it('returns each node once', async () => { + const series = (node: string) => ({ metric: { labels: { node } } }); + const answering = driver(async () => ({ + resultType: 'vector', + result: [series('n1'), series('n1'), series('n2')] + })); + const nodes = await fetchModelNodes(answering, 'ws-a', 'm1'); + expect(nodes?.slice().sort()).toEqual(['n1', 'n2']); + }); +}); + +describe('fetchCombinedInstantOrNull', () => { + const driver = (instantQuery: () => Promise) => + ({ instantQuery }) as unknown as PrometheusDriver; + const queries = { restarts: 'a', ooms: 'b' }; + + it('resolves null when the query fails, so callers can show unknown rather than 0', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const failing = driver(async () => { + throw new Error('HTTP 403'); + }); + await expect(fetchCombinedInstantOrNull(failing, queries)).resolves.toBeNull(); + expect(warn).toHaveBeenCalled(); + await expect(fetchCombinedInstant(failing, queries)).resolves.toEqual({ + restarts: [], + ooms: [] + }); + warn.mockRestore(); + }); + + it('splits the combined answer by name, without the tag', async () => { + const answering = driver(async () => ({ + resultType: 'vector', + result: [{ metric: { labels: { __series__: 'ooms', pod: 'p1' } } }] + })); + const result = await fetchCombinedInstantOrNull(answering, queries); + expect(result?.restarts).toEqual([]); + expect(result?.ooms.map((v) => v.metric.labels)).toEqual([{ pod: 'p1' }]); + }); +}); diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 8e03c003..94469eba 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -1,13 +1,198 @@ -import { InstantVector, type PrometheusDriver, RangeVector } from 'prometheus-query'; +import { InstantVector, PrometheusDriver, QueryResult, RangeVector } from 'prometheus-query'; import type { ChartConfig } from '$lib/components/ui/chart/index.js'; import { m } from '$lib/messages'; +/* + * The OtterScale server answers metrics at three scopes. + * + * Cluster scope (`/proxy//prometheus`) takes any query, but only from + * viewers with cluster metrics access (cluster-admin, or a binding to + * otterscale:metrics-cluster-view); everyone else gets 403 whatever they ask. + * + * Namespace scope (`/proxy//prometheus-tenancy?namespace=`) takes + * any query too, from anyone who may get pods in , rewritten by the server + * to namespace="" the way prom-label-proxy does it. A namespace-scoped page + * therefore only works with metrics whose `namespace` label is the tenant's — + * see docs/metrics.md. + * + * Shared scope (`/proxy//prometheus-shared/`) takes no PromQL: + * the server runs one of its own named queries — cluster-wide numbers that + * name no workload, per GPU card, per host, or a total — for the namespace + * scope's callers as well as cluster viewers. See sharedMetrics. + */ + +const PROXY_HEADERS = { 'x-proxy-target': 'api' }; + +/** `origin` is for tests; the browser uses same-origin paths. */ +export function clusterPrometheusDriver(cluster: string, origin = ''): PrometheusDriver { + return new PrometheusDriver({ + endpoint: `${origin}/proxy/${cluster}/prometheus`, + baseURL: '/api/v1', + headers: PROXY_HEADERS + }); +} + +export function namespacePrometheusDriver( + cluster: string, + namespace: string, + origin = '' +): PrometheusDriver { + return new PrometheusDriver({ + endpoint: `${origin}/proxy/${cluster}/prometheus-tenancy`, + baseURL: '/api/v1', + headers: PROXY_HEADERS, + // prometheus-query has no option for extra query parameters. It always + // hands axios `params` as URLSearchParams, so the namespace is added there + // — in the URL, where the server reads it, even when a body is sent. + requestInterceptor: { + onFulfilled: (config) => { + const params = + config.params instanceof URLSearchParams + ? config.params + : new URLSearchParams(config.params ?? {}); + params.set('namespace', namespace); + config.params = params; + return config; + } + } + }); +} + +/** + * For pages that show one Workspace's namespace. Viewers with cluster metrics + * access keep cluster scope, and with it the admin-only "all namespaces" + * choice; everyone else gets namespace scope, or nothing when the Workspace + * has no namespace to show. + */ +export function workspacePrometheusDriver( + cluster: string, + namespace: string, + canViewClusterMetrics: boolean +): PrometheusDriver | null { + if (canViewClusterMetrics) return clusterPrometheusDriver(cluster); + return namespace ? namespacePrometheusDriver(cluster, namespace) : null; +} + +export type DashboardBody = 'charts' | 'loading' | 'no-namespace' | 'empty'; + +/** + * What a Workspace dashboard shows where its charts go. The driver is made on + * mount, so until then the page is loading. A viewer without cluster scope + * gets no driver when the Workspace's namespace is unknown — usually because + * the Workspace could not be read — which is a state to explain, not to spin on. + */ +export function dashboardBody(state: { + hasDriver: boolean; + initialized: boolean; + cluster: string; + namespace: string; +}): DashboardBody { + if (state.hasDriver) return 'charts'; + if (!state.cluster) return 'empty'; + if (!state.initialized) return 'loading'; + return state.namespace ? 'empty' : 'no-namespace'; +} + +/** The server's named cluster-wide queries (otterscale internal/handler/proxy_shared_catalog.go). */ +export type SharedQueryName = + | 'gpu-util' + | 'gpu-fb-used' + | 'gpu-fb-total' + | 'gpu-inventory' + | 'vgpu-memory-limit' + | 'vgpu-memory-used' + | 'host-gpu-fb-used' + | 'host-gpu-fb-total' + | 'host-gpu-count' + | 'gpu-xid-total' + | 'gpu-xid-by-gpu' + | 'kubevirt-handlers' + | 'kubevirt-controllers' + | 'kubevirt-controllers-ready' + | 'ceph-total-bytes' + | 'gpu-allocated-memory' + | 'gpu-allocated-cores' + | 'gpu-shared-count'; + +/** Values for the one repeatable parameter a shared query takes, if any. */ +export type SharedParams = { uuid?: string[]; hostname?: string[] }; + +export interface SharedMetrics { + instant(name: SharedQueryName, params?: SharedParams, time?: Date): Promise; + range( + name: SharedQueryName, + params: SharedParams, + start: Date, + end: Date, + step: number + ): Promise; +} + +/** + * Shared scope: cluster-wide numbers that name no workload — per GPU card, per + * host, or a total — for Workspace members and cluster viewers alike. The + * server holds the PromQL; this sends a name and parameters. `namespace` is the + * Workspace's and only proves membership, so an admin's "all namespaces" choice + * does not change it; it is left out when the Workspace has none, and then only + * cluster viewers get an answer. Failures reject with the server's message. + */ +export function sharedMetrics(cluster: string, namespace: string, origin = ''): SharedMetrics { + async function get(name: SharedQueryName, search: URLSearchParams): Promise { + if (namespace) search.set('namespace', namespace); + const response = await fetch(`${origin}/proxy/${cluster}/prometheus-shared/${name}?${search}`, { + headers: PROXY_HEADERS + }); + const body = (await response.json().catch(() => null)) as { + status?: string; + data?: object; + error?: string; + } | null; + if (!response.ok || body?.status !== 'success' || !body.data) { + throw new Error(`${name}: ${body?.error ?? `HTTP ${response.status}`}`); + } + return QueryResult.fromJSON(body.data); + } + + function searchFor(params: SharedParams = {}): URLSearchParams { + const search = new URLSearchParams(); + for (const uuid of params.uuid ?? []) search.append('uuid', uuid); + for (const hostname of params.hostname ?? []) search.append('hostname', hostname); + return search; + } + + const seconds = (date: Date) => String(date.getTime() / 1000); + + return { + instant(name, params, time) { + const search = searchFor(params); + if (time) search.set('time', seconds(time)); + return get(name, search); + }, + range(name, params, start, end, step) { + const search = searchFor(params); + search.set('start', seconds(start)); + search.set('end', seconds(end)); + search.set('step', String(step)); + return get(name, search); + } + }; +} + /** Escape a value for use inside PromQL double-quoted string literals (e.g. `namespace="..."`). */ export function escapePromqlStringLiteral(value: string): string { return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n'); } +/** + * `{namespace=""}`, or nothing for an admin's "all namespaces" (the empty + * choice): namespace="" would match only series without the label. At + * namespace scope the server pins the tenant's namespace either way. + */ +export function namespaceSelector(namespace: string): string { + return namespace ? `{namespace="${escapePromqlStringLiteral(namespace)}"}` : ''; +} + /** * Restrict a per-pod KSM selector to non-terminated pods — the set `kubectl describe node` * counts. KSM keeps emitting requests/limits for Succeeded/Failed pods until the object is @@ -397,16 +582,13 @@ export async function fetchCombinedFlattenedRange( /** * Run multiple named instant queries as a SINGLE HTTP request. * Returns a record mapping each query name to its InstantVector list (after the - * `__series__` tag has been stripped, so callers see the original label sets). - * - * On error, every named slot is set to `[]` so callers can deconstruct without - * needing a try/catch per query — matching the lenient `instantNumber` / - * `instantPerPod` pattern used in the dashboard. + * `__series__` tag has been stripped, so callers see the original label sets), + * or null when the request failed, so callers can show "unknown" rather than 0. */ -export async function fetchCombinedInstant( +export async function fetchCombinedInstantOrNull( client: PrometheusDriver, queries: Record -): Promise> { +): Promise | null> { const result: Record = {}; for (const name of Object.keys(queries)) result[name] = []; const combined = Object.entries(queries) @@ -423,41 +605,30 @@ export async function fetchCombinedInstant( vector.metric.labels = rest; result[name].push(vector); } - } catch { - // leave all slots empty + } catch (error) { + console.warn('Failed to run combined instant query:', error); + return null; } return result; } /** - * Build a sub-expression that yields one synthetic row per K8s node a given vLLM - * model's pods occupy, with the `node` label renamed to `hostname` so the result - * can be intersected against DCGM / host-level metrics via `and on(hostname)`. - * - * Lets DCGM queries scope to the model's hosts in a SINGLE PromQL request — no - * preceding round-trip is needed to enumerate node names client-side. + * {@link fetchCombinedInstantOrNull}, with every named slot set to `[]` on + * error so callers can deconstruct without needing a try/catch per query — + * matching the lenient `instantNumber` / `instantPerPod` pattern used in the + * dashboard. */ -export function vllmModelHostnamesSelector( - namespace: string | undefined, - selectedModel: string -): string { - const ns = (namespace ?? '').trim(); - const nsSel = ns ? `namespace="${escapePromqlStringLiteral(ns)}"` : ''; - const podInfoSelector = nsSel ? `{${nsSel}}` : ''; - const vllmSelector = vllmMetricWithSelector('vllm:kv_cache_usage_perc', namespace, selectedModel); - return ( - `group by(hostname) (` + - `label_replace(` + - `kube_pod_info${podInfoSelector}` + - ` * on(namespace, pod) group_left() ` + - `group by(namespace, pod) (${vllmSelector}),` + - ` "hostname", "$1", "node", "(.+)"))` - ); +export async function fetchCombinedInstant( + client: PrometheusDriver, + queries: Record +): Promise> { + const result = await fetchCombinedInstantOrNull(client, queries); + return result ?? Object.fromEntries(Object.keys(queries).map((name) => [name, []])); } /** * Label selector scoping DCGM (GPU exporter) series to one Kubernetes node. DCGM's - * `hostname` label equals the K8s node name (as vllmModelHostnamesSelector also assumes). + * `hostname` label equals the K8s node name (as the model dashboard's GPU panels also assume). * Exact match, not regex: node names can contain dots and would otherwise cross-match. */ export function dcgmNodeSelector(nodeName: string): string { @@ -572,13 +743,14 @@ export function classifyGpuGovernance(allocated?: number, used?: number): GpuGov * Joins `kube_pod_info` against `vllm:kv_cache_usage_perc` so we only keep pods * that belong to `selectedModel`, then returns the distinct `node` label values. * Used to scope DCGM/GPU queries (which lack pod-level labels) to the nodes the - * model actually occupies. + * model actually occupies. Resolves null when the lookup failed, so callers can + * tell 'unknown' from 'no GPU'. */ export async function fetchModelNodes( client: PrometheusDriver, namespace: string | undefined, selectedModel: string -): Promise { +): Promise { const ns = (namespace ?? '').trim(); const nsSel = ns ? `namespace="${escapePromqlStringLiteral(ns)}"` : ''; const podInfoSelector = nsSel ? `{${nsSel}}` : ''; @@ -597,7 +769,7 @@ export async function fetchModelNodes( } return Array.from(nodes); } catch { - return []; + return null; } } diff --git a/src/lib/server/metrics-access.spec.ts b/src/lib/server/metrics-access.spec.ts new file mode 100644 index 00000000..165fcc6c --- /dev/null +++ b/src/lib/server/metrics-access.spec.ts @@ -0,0 +1,114 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { checkClusterMetricsAccess, CLUSTER_METRICS_ACCESS } from './metrics-access'; + +type Step = () => Response | Error; + +/** A fetch that answers with each step in turn; the last one repeats. */ +function fakeFetch(...steps: Step[]) { + let call = 0; + return vi.fn(async () => { + const answer = steps[Math.min(call++, steps.length - 1)](); + if (answer instanceof Error) throw answer; + return answer; + }) as unknown as typeof fetch & ReturnType; +} + +const allowed = + (value: boolean): Step => + () => + new Response(JSON.stringify({ object: { status: { allowed: value } } }), { status: 200 }); +const status = + (code: number): Step => + () => + new Response('', { status: code }); + +describe('checkClusterMetricsAccess', () => { + it('asks the hub for the exact attribute the server checks', async () => { + const f = fakeFetch(allowed(true)); + await expect(checkClusterMetricsAccess(f, 'c1')).resolves.toBe('allowed'); + + const [url, init] = f.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('/otterscale.resource.v1.ResourceService/Create'); + const body = JSON.parse(init.body as string); + expect(body).toMatchObject({ + cluster: 'c1', + group: 'authorization.k8s.io', + version: 'v1', + resource: 'selfsubjectaccessreviews' + }); + const manifest = JSON.parse(atob(body.manifest)); + expect(manifest.kind).toBe('SelfSubjectAccessReview'); + expect(manifest.spec.resourceAttributes).toEqual(CLUSTER_METRICS_ACCESS); + expect(CLUSTER_METRICS_ACCESS).toEqual({ + verb: 'get', + group: 'monitoring.coreos.com', + resource: 'prometheuses', + subresource: 'api', + namespace: 'monitoring', + name: 'prometheus-stack-kube-prom-prometheus' + }); + }); + + it('takes a denial at its word: no retry, no log', async () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const f = fakeFetch(allowed(false)); + await expect(checkClusterMetricsAccess(f, 'c1')).resolves.toBe('denied'); + expect(f).toHaveBeenCalledTimes(1); + + const odd = fakeFetch(() => new Response('{}', { status: 200 })); + await expect(checkClusterMetricsAccess(odd, 'c1')).resolves.toBe('denied'); + expect(odd).toHaveBeenCalledTimes(1); + expect(log).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + }); + + it('asks once more after a failure', async () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const f = fakeFetch(status(503), allowed(true)); + await expect(checkClusterMetricsAccess(f, 'c1')).resolves.toBe('allowed'); + expect(f).toHaveBeenCalledTimes(2); + + const thrown = fakeFetch(() => new Error('blip'), allowed(true)); + await expect(checkClusterMetricsAccess(thrown, 'c1')).resolves.toBe('allowed'); + expect(thrown).toHaveBeenCalledTimes(2); + expect(log).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + }); + + it('is unknown after two failures, and logs why once', async () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const f = fakeFetch(status(503)); + await expect(checkClusterMetricsAccess(f, 'c1')).resolves.toBe('unknown'); + expect(f).toHaveBeenCalledTimes(2); + expect(log).toHaveBeenCalledTimes(1); + expect(String(log.mock.calls[0][0])).toContain('c1'); + expect(log.mock.calls[0][1]).toBe('HTTP 503'); + + log.mockClear(); + const down = new Error('down'); + await expect( + checkClusterMetricsAccess( + fakeFetch(() => down), + 'c1' + ) + ).resolves.toBe('unknown'); + expect(log).toHaveBeenCalledTimes(1); + expect(log.mock.calls[0][1]).toBe(down); + + log.mockClear(); + const garbled = fakeFetch(() => new Response('not json', { status: 200 })); + await expect(checkClusterMetricsAccess(garbled, 'c1')).resolves.toBe('unknown'); + expect(log).toHaveBeenCalledTimes(1); + } finally { + log.mockRestore(); + } + }); +}); diff --git a/src/lib/server/metrics-access.ts b/src/lib/server/metrics-access.ts new file mode 100644 index 00000000..697750a7 --- /dev/null +++ b/src/lib/server/metrics-access.ts @@ -0,0 +1,71 @@ +import type { ClusterMetricsAccess } from '$lib/cluster-metrics-access'; + +/** + * The attribute the OtterScale server checks before answering cluster-scope + * PromQL (otterscale internal/core/access.go, ClusterMetricsAccess). Kept + * identical so the dashboard shows a viewer exactly the pages the server will + * serve them. + */ +export const CLUSTER_METRICS_ACCESS = { + verb: 'get', + group: 'monitoring.coreos.com', + resource: 'prometheuses', + subresource: 'api', + namespace: 'monitoring', + name: 'prometheus-stack-kube-prom-prometheus' +} as const; + +type AccessReviewResponse = { object?: { status?: { allowed?: boolean } } }; + +/** A failed check is asked once more: a tunnel blip should not hide pages from an admin. */ +const ATTEMPTS = 2; + +/** + * Whether the signed-in user may read cluster-scope metrics on `cluster`, + * asked of that cluster as them (a SelfSubjectAccessReview through the hub). + * + * The cluster's answer is final, so a denial is not asked again. A failure + * (hub or tunnel error, or an unreadable answer) is asked once more and then + * reads as unknown, which is logged; a denial is not. + */ +export async function checkClusterMetricsAccess( + svelteFetch: typeof fetch, + cluster: string +): Promise { + const init: RequestInit = { + method: 'POST', + headers: { 'x-proxy-target': 'api', 'Content-Type': 'application/json' }, + body: JSON.stringify({ + cluster, + group: 'authorization.k8s.io', + version: 'v1', + resource: 'selfsubjectaccessreviews', + manifest: btoa( + JSON.stringify({ + apiVersion: 'authorization.k8s.io/v1', + kind: 'SelfSubjectAccessReview', + spec: { resourceAttributes: CLUSTER_METRICS_ACCESS } + }) + ) + }) + }; + + let failure: unknown; + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + try { + const response = await svelteFetch('/otterscale.resource.v1.ResourceService/Create', init); + if (response.ok) { + const data = (await response.json()) as AccessReviewResponse; + return data.object?.status?.allowed === true ? 'allowed' : 'denied'; + } + failure = `HTTP ${response.status}`; + } catch (error) { + failure = error; + } + } + console.error( + `Failed to check cluster metrics access on ${cluster} (${ATTEMPTS} attempts):`, + failure + ); + return 'unknown'; +} diff --git a/src/routes/(auth)/+layout.server.ts b/src/routes/(auth)/+layout.server.ts index 6df737fc..1c47116a 100644 --- a/src/routes/(auth)/+layout.server.ts +++ b/src/routes/(auth)/+layout.server.ts @@ -1,6 +1,8 @@ import { redirect } from '@sveltejs/kit'; import { env } from '$env/dynamic/private'; +import type { ClusterMetricsAccess } from '$lib/cluster-metrics-access'; +import { checkClusterMetricsAccess } from '$lib/server/metrics-access'; import type { LayoutServerLoad } from './$types'; @@ -23,15 +25,25 @@ export const load: LayoutServerLoad = async ({ locals, params, fetch }) => { } let isClusterAdmin = false; - try { - if (params.cluster && user.sub) { - isClusterAdmin = await verifyClusterAdminStatus(fetch, params.cluster, user.sub); - } - } catch (error) { - console.error('Failed to verify admin status:', error); + let clusterMetricsAccess: ClusterMetricsAccess = 'denied'; + if (params.cluster && user.sub) { + [isClusterAdmin, clusterMetricsAccess] = await Promise.all([ + verifyClusterAdminStatus(fetch, params.cluster, user.sub).catch((error) => { + console.error('Failed to verify admin status:', error); + return false; + }), + checkClusterMetricsAccess(fetch, params.cluster) + ]); } - return { user, isClusterAdmin, agentValuesInsecureTLS }; + return { + user, + isClusterAdmin, + clusterMetricsAccess, + // What pages draw from: only a yes counts, so unknown draws the regular view. + canViewClusterMetrics: clusterMetricsAccess === 'allowed', + agentValuesInsecureTLS + }; }; async function verifyClusterAdminStatus( diff --git a/src/routes/(auth)/+layout.svelte b/src/routes/(auth)/+layout.svelte index 9b796cb6..728c3665 100644 --- a/src/routes/(auth)/+layout.svelte +++ b/src/routes/(auth)/+layout.svelte @@ -15,6 +15,7 @@ import LayersIcon from '@lucide/svelte/icons/layers'; import LayoutGridIcon from '@lucide/svelte/icons/layout-grid'; import NetworkIcon from '@lucide/svelte/icons/network'; + import TriangleAlertIcon from '@lucide/svelte/icons/triangle-alert'; import UserStarIcon from '@lucide/svelte/icons/user-star'; import { type Link, LinkService } from '@otterscale/api/link/v1'; import { ResourceService } from '@otterscale/api/resource/v1'; @@ -36,6 +37,7 @@ } from '$lib/components/layout'; import ImportCluster from '$lib/components/layout/import-cluster-external.svelte'; import RegisteClusterTrigger from '$lib/components/layout/registe-cluster-trigger.svelte'; + import * as Alert from '$lib/components/ui/alert'; import * as Breadcrumb from '$lib/components/ui/breadcrumb'; import { Button } from '$lib/components/ui/button'; import * as DropdownMenu from '$lib/components/ui/dropdown-menu'; @@ -247,15 +249,19 @@ title: m.dashboard(), icon: GaugeIcon, items: [ - { - title: m.cluster(), - url: page.params.workspace - ? resolve('/(auth)/[cluster]/[workspace]/dashboard/cluster', { - cluster: activeCluster, - workspace: page.params.workspace - }) - : '' - }, + ...(data.canViewClusterMetrics + ? [ + { + title: m.cluster(), + url: page.params.workspace + ? resolve('/(auth)/[cluster]/[workspace]/dashboard/cluster', { + cluster: activeCluster, + workspace: page.params.workspace + }) + : '' + } + ] + : []), { title: m.model(), url: page.params.workspace @@ -274,7 +280,7 @@ }) : '' }, - ...(hasRookCeph + ...(hasRookCeph && data.canViewClusterMetrics ? [ { title: m.storage(), @@ -864,6 +870,12 @@
      + {#if data.clusterMetricsAccess === 'unknown'} + + + {m.cluster_metrics_access_unknown()} + + {/if} {@render children()}
      diff --git a/src/routes/(auth)/[cluster]/[workspace]/dashboard/cluster/+page.ts b/src/routes/(auth)/[cluster]/[workspace]/dashboard/cluster/+page.ts new file mode 100644 index 00000000..7ee4b2c2 --- /dev/null +++ b/src/routes/(auth)/[cluster]/[workspace]/dashboard/cluster/+page.ts @@ -0,0 +1,22 @@ +import { redirect } from '@sveltejs/kit'; + +import { resolve } from '$app/paths'; +import { clusterDashboardRedirect } from '$lib/cluster-metrics-access'; + +import type { PageLoad } from './$types'; + +// Cluster-wide dashboards are for viewers with cluster metrics access; the +// server answers everyone else 403. A bookmarked URL lands on the overview +// instead of a page of failing charts — unless the check itself failed, in +// which case the layout says so and the viewer stays. +export const load: PageLoad = async ({ parent, params }) => { + const { clusterMetricsAccess } = await parent(); + const target = clusterDashboardRedirect( + clusterMetricsAccess, + resolve('/(auth)/[cluster]/[workspace]/dashboard/overview', { + cluster: params.cluster, + workspace: params.workspace + }) + ); + if (target) redirect(307, target); +}; diff --git a/src/routes/(auth)/[cluster]/[workspace]/dashboard/compute/+page.svelte b/src/routes/(auth)/[cluster]/[workspace]/dashboard/compute/+page.svelte index 970bc53e..7c9aa945 100644 --- a/src/routes/(auth)/[cluster]/[workspace]/dashboard/compute/+page.svelte +++ b/src/routes/(auth)/[cluster]/[workspace]/dashboard/compute/+page.svelte @@ -22,6 +22,14 @@ const isClusterAdmin = $derived(page.data.isClusterAdmin === true); -{#key cluster} - + +{#key cluster + currentWorkspace + defaultNamespace} + {/key} diff --git a/src/routes/(auth)/[cluster]/[workspace]/dashboard/model/+page.svelte b/src/routes/(auth)/[cluster]/[workspace]/dashboard/model/+page.svelte index 9b3bc861..69e93093 100644 --- a/src/routes/(auth)/[cluster]/[workspace]/dashboard/model/+page.svelte +++ b/src/routes/(auth)/[cluster]/[workspace]/dashboard/model/+page.svelte @@ -22,6 +22,14 @@ const isClusterAdmin = $derived(page.data.isClusterAdmin === true); -{#key cluster} - + +{#key cluster + currentWorkspace + defaultNamespace} + {/key} diff --git a/src/routes/(auth)/[cluster]/[workspace]/dashboard/overview/+page.svelte b/src/routes/(auth)/[cluster]/[workspace]/dashboard/overview/+page.svelte index 7141254a..66db01ac 100644 --- a/src/routes/(auth)/[cluster]/[workspace]/dashboard/overview/+page.svelte +++ b/src/routes/(auth)/[cluster]/[workspace]/dashboard/overview/+page.svelte @@ -1,6 +1,11 @@ {#key cluster + workspace + namespace} - + {/key} diff --git a/src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/+page.ts b/src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/+page.ts new file mode 100644 index 00000000..7ee4b2c2 --- /dev/null +++ b/src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/+page.ts @@ -0,0 +1,22 @@ +import { redirect } from '@sveltejs/kit'; + +import { resolve } from '$app/paths'; +import { clusterDashboardRedirect } from '$lib/cluster-metrics-access'; + +import type { PageLoad } from './$types'; + +// Cluster-wide dashboards are for viewers with cluster metrics access; the +// server answers everyone else 403. A bookmarked URL lands on the overview +// instead of a page of failing charts — unless the check itself failed, in +// which case the layout says so and the viewer stays. +export const load: PageLoad = async ({ parent, params }) => { + const { clusterMetricsAccess } = await parent(); + const target = clusterDashboardRedirect( + clusterMetricsAccess, + resolve('/(auth)/[cluster]/[workspace]/dashboard/overview', { + cluster: params.cluster, + workspace: params.workspace + }) + ); + if (target) redirect(307, target); +};