From 084627f23d9a6fa22e24ad938e8ecb5437846c97 Mon Sep 17 00:00:00 2001 From: hstsai Date: Thu, 8 Oct 2026 19:18:29 +0800 Subject: [PATCH 01/41] feat(dashboard): namespace-scoped metrics for regular users Viewers without cluster metrics access query /prometheus-tenancy for their workspace's namespace, no longer see the cluster and storage dashboards, and the overview drops the Ceph capacity check. Co-Authored-By: Claude Sonnet 5.5 --- .../dashboard/workspace/index.svelte | 17 ++--- .../workspace/overview/storage.svelte | 8 ++- src/lib/prometheus.spec.ts | 66 +++++++++++++++++- src/lib/prometheus.ts | 67 ++++++++++++++++++- src/lib/server/metrics-access.spec.ts | 52 ++++++++++++++ src/lib/server/metrics-access.ts | 53 +++++++++++++++ src/routes/(auth)/+layout.server.ts | 18 +++-- src/routes/(auth)/+layout.svelte | 24 ++++--- .../[workspace]/dashboard/cluster/+page.ts | 21 ++++++ .../dashboard/overview/+page.svelte | 6 +- .../[workspace]/dashboard/storage/+page.ts | 21 ++++++ 11 files changed, 320 insertions(+), 33 deletions(-) create mode 100644 src/lib/server/metrics-access.spec.ts create mode 100644 src/lib/server/metrics-access.ts create mode 100644 src/routes/(auth)/[cluster]/[workspace]/dashboard/cluster/+page.ts create mode 100644 src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/+page.ts diff --git a/src/lib/components/dashboard/workspace/index.svelte b/src/lib/components/dashboard/workspace/index.svelte index 67cb70c7..3bfce3db 100644 --- a/src/lib/components/dashboard/workspace/index.svelte +++ b/src/lib/components/dashboard/workspace/index.svelte @@ -1,6 +1,6 @@ {#key cluster + workspace + namespace} - + {/key} diff --git a/src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/+page.ts b/src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/+page.ts new file mode 100644 index 00000000..caa14bb9 --- /dev/null +++ b/src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/+page.ts @@ -0,0 +1,21 @@ +import { redirect } from '@sveltejs/kit'; + +import { resolve } from '$app/paths'; + +import type { PageLoad } from './$types'; + +// Cluster-wide dashboards are for viewers with cluster metrics access; the +// server answers everyone else 403. A bookmarked URL lands on the overview +// instead of a page of failing charts. +export const load: PageLoad = async ({ parent, params }) => { + const { canViewClusterMetrics } = await parent(); + if (!canViewClusterMetrics) { + redirect( + 307, + resolve('/(auth)/[cluster]/[workspace]/dashboard/overview', { + cluster: params.cluster, + workspace: params.workspace + }) + ); + } +}; From d03b4ff483d999a6063f9d026769be0d27e9d2fa Mon Sep 17 00:00:00 2001 From: hstsai Date: Thu, 8 Oct 2026 19:32:26 +0800 Subject: [PATCH 02/41] feat(dashboard): namespace-scoped compute page Regular users query the Workspace's namespace through the tenancy proxy. VM metrics are selected on namespace rather than exported_namespace: the kubevirt PodMonitor now keeps the VMI's own namespace label, which is the one namespace scope enforces. The KubeVirt controller and worker tiles are cluster scope and are dropped for viewers without cluster metrics access, so their queries are never sent. Co-Authored-By: Claude Opus 5.5 --- .../compute/analytics/vm-picker.svelte | 2 +- .../components/dashboard/compute/index.svelte | 23 +++++++++--------- .../dashboard/compute/overview/cpu.svelte | 6 ++--- .../compute/overview/instance.svelte | 2 +- .../dashboard/compute/overview/memory.svelte | 2 +- .../compute/overview/network-traffic.svelte | 4 ++-- .../compute/overview/throughput.svelte | 4 ++-- .../compute/overview/virtual-machine.svelte | 2 +- .../compute/overview/widgets.spec.ts | 17 +++++++++++++ .../dashboard/compute/overview/widgets.ts | 9 +++++++ .../dashboard/namespace-scope.spec.ts | 24 +++++++++++++++++++ .../dashboard/compute/+page.svelte | 12 ++++++++-- 12 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 src/lib/components/dashboard/compute/overview/widgets.spec.ts create mode 100644 src/lib/components/dashboard/namespace-scope.spec.ts diff --git a/src/lib/components/dashboard/compute/analytics/vm-picker.svelte b/src/lib/components/dashboard/compute/analytics/vm-picker.svelte index 381bea85..9898b3f0 100644 --- a/src/lib/components/dashboard/compute/analytics/vm-picker.svelte +++ b/src/lib/components/dashboard/compute/analytics/vm-picker.svelte @@ -74,7 +74,7 @@ isLoaded = false; try { const response = await prometheusDriver.instantQuery( - `kubevirt_vmi_info{` + (ns ? `exported_namespace="${ns}"` : '') + `}` + `kubevirt_vmi_info{` + (ns ? `namespace="${ns}"` : '') + `}` ); if (cancelled) return; const series = (response.result ?? []) as { diff --git a/src/lib/components/dashboard/compute/index.svelte b/src/lib/components/dashboard/compute/index.svelte index b068b8a1..a5485af2 100644 --- a/src/lib/components/dashboard/compute/index.svelte +++ b/src/lib/components/dashboard/compute/index.svelte @@ -1,5 +1,5 @@ -{#key cluster} - + +{#key cluster + currentWorkspace + defaultNamespace} + {/key} From 22c5a97538aed6b7d315a1ac33758e4f47a7f620 Mon Sep 17 00:00:00 2001 From: hstsai Date: Thu, 8 Oct 2026 19:32:26 +0800 Subject: [PATCH 03/41] feat(dashboard): namespace-scoped model page Regular users query the Workspace's namespace through the tenancy proxy. GPU memory and XID come from DCGM, which carries gpu-operator's namespace, so viewers without cluster metrics access see neither and send neither query; the vLLM, kube-state-metrics and cAdvisor panels are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../model/analytics/b-pod-health.svelte | 122 ++++++++++-------- .../dashboard/model/analytics/index.svelte | 24 ++-- .../components/dashboard/model/index.svelte | 20 +-- .../[workspace]/dashboard/model/+page.svelte | 12 +- 4 files changed, 103 insertions(+), 75 deletions(-) diff --git a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte index 58a00964..67b29bd6 100644 --- a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte +++ b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte @@ -23,11 +23,14 @@ prometheusDriver, namespace, selectedModel, + canViewClusterMetrics = false, isReloading = $bindable() }: { prometheusDriver: PrometheusDriver; namespace: string | undefined; selectedModel: string; + /** XID comes from DCGM, which carries the exporter's namespace: cluster scope only. */ + canViewClusterMetrics?: boolean; isReloading: boolean; } = $props(); @@ -111,6 +114,32 @@ return exitcode === 135; } + /** XID errors on the GPUs of the nodes the model runs on, from DCGM. */ + async function fetchXid(nodes: string[]): Promise<{ total: number; byGpu: GpuEntry[] }> { + if (nodes.length === 0) return { total: 0, byGpu: [] }; + const regex = escapePromqlStringLiteral(nodes.map(regexEscape).join('|')); + const selector = `{hostname=~"${regex}"}`; + const xid = await fetchCombinedInstant(prometheusDriver, { + total: `sum(increase(DCGM_FI_DEV_XID_ERRORS${selector}[24h]))`, + perGpu: `sum by(hostname, gpu) (increase(DCGM_FI_DEV_XID_ERRORS${selector}[24h])) > 0` + }); + const total = Math.round(scalarFromVectors(xid.total)); + const byGpu = xid.perGpu + .map((v) => { + const labels = v.metric.labels as Record; + const value = Number(v.value?.value); + if (!Number.isFinite(value)) return null; + return { + hostname: labels.hostname ?? '(unknown)', + gpu: labels.gpu ?? '?', + value + }; + }) + .filter((x): x is GpuEntry => x !== null) + .sort((a, b) => b.value - a.value); + return { total, byGpu }; + } + async function fetch() { const cs = containerSelector(); const join = modelJoin(); @@ -135,31 +164,11 @@ const nodesPromise = fetchModelNodes(prometheusDriver, namespace, selectedModel); // Kick off XID as soon as nodes resolves, in parallel with the main batch — - // it only needs `nodes`, not the full set of cluster-wide queries. - const xidPromise = nodesPromise.then(async (nodes) => { - if (nodes.length === 0) return { total: 0, byGpu: [] as GpuEntry[] }; - const regex = escapePromqlStringLiteral(nodes.map(regexEscape).join('|')); - const selector = `{hostname=~"${regex}"}`; - const xid = await fetchCombinedInstant(prometheusDriver, { - total: `sum(increase(DCGM_FI_DEV_XID_ERRORS${selector}[24h]))`, - perGpu: `sum by(hostname, gpu) (increase(DCGM_FI_DEV_XID_ERRORS${selector}[24h])) > 0` - }); - const total = Math.round(scalarFromVectors(xid.total)); - const byGpu = xid.perGpu - .map((v) => { - const labels = v.metric.labels as Record; - const value = Number(v.value?.value); - if (!Number.isFinite(value)) return null; - return { - hostname: labels.hostname ?? '(unknown)', - gpu: labels.gpu ?? '?', - value - }; - }) - .filter((x): x is GpuEntry => x !== null) - .sort((a, b) => b.value - a.value); - return { total, byGpu }; - }); + // it only needs `nodes`, not the full set of cluster-wide queries. Viewers without + // cluster metrics access never see the tile, so they never send the DCGM query. + const xidPromise = canViewClusterMetrics + ? nodesPromise.then(fetchXid) + : Promise.resolve({ total: 0, byGpu: [] as GpuEntry[] }); const [main, nodes, xid] = await Promise.all([ fetchCombinedInstant(prometheusDriver, mainQueries), @@ -251,7 +260,8 @@ {:else} + would squeeze the labels. Each tile's colour is its own status; there is no roll-up. + Without cluster metrics access there are five: XID is DCGM, cluster scope only. -->
@@ -303,35 +313,37 @@ {/if} - - -
-

- {nodeCount === 0 ? '—' : xidTotal} -

-

- {m.metric_xid()} -

-
- - {#if nodeCount === 0} - -

{m.no_gpu_for_model()}

- - {:else if xidByGpu.length > 0} - -
    - {#each xidByGpu as g (g.hostname + g.gpu)} -
  • - {g.hostname} - GPU{g.gpu} - {Math.round(g.value)} -
    • - {/each} -
- - {/if} - + {#if canViewClusterMetrics} + + +
+

+ {nodeCount === 0 ? '—' : xidTotal} +

+

+ {m.metric_xid()} +

+
+ + {#if nodeCount === 0} + +

{m.no_gpu_for_model()}

+ + {:else if xidByGpu.length > 0} + +
    + {#each xidByGpu as g (g.hostname + g.gpu)} +
  • + {g.hostname} + GPU{g.gpu} + {Math.round(g.value)} +
    • + {/each} +
+ + {/if} + + {/if} diff --git a/src/lib/components/dashboard/model/analytics/index.svelte b/src/lib/components/dashboard/model/analytics/index.svelte index 88b9dad1..edd48b9f 100644 --- a/src/lib/components/dashboard/model/analytics/index.svelte +++ b/src/lib/components/dashboard/model/analytics/index.svelte @@ -31,6 +31,7 @@ start, end, endIsNow, + canViewClusterMetrics = false, isReloading = $bindable() }: { cluster: string; @@ -40,6 +41,8 @@ start: Date; end: Date; endIsNow: boolean; + /** GPU (DCGM) panels are cluster scope: the exporter's namespace, not the tenant's. */ + canViewClusterMetrics?: boolean; isReloading?: boolean; } = $props(); @@ -105,6 +108,7 @@ prometheusDriver={client} selectedModel={modelFilter} isReloading={isReloading ?? false} + {canViewClusterMetrics} /> - + {#if canViewClusterMetrics} + + {/if}
{/if} diff --git a/src/lib/components/dashboard/model/index.svelte b/src/lib/components/dashboard/model/index.svelte index a8c270b3..fd4b714c 100644 --- a/src/lib/components/dashboard/model/index.svelte +++ b/src/lib/components/dashboard/model/index.svelte @@ -1,6 +1,6 @@ -{#key cluster} - + +{#key cluster + currentWorkspace + defaultNamespace} + {/key} From d516b4f9a4fee5430f1bb45dc82c704ef197723c Mon Sep 17 00:00:00 2001 From: hstsai Date: Thu, 8 Oct 2026 19:32:26 +0800 Subject: [PATCH 04/41] feat(dashboard): GPU inventory dialogs for cluster metrics viewers only The GPU allocation dialog and the GPU resources modal read nodes, DCGM and HAMi at cluster scope, where a regular user only gets 403. They now render, and query, only for viewers with cluster metrics access, and the actions menus drop the GPU allocation row with them. Co-Authored-By: Claude Opus 5.5 --- .../gpu-allocation-dialog.svelte | 130 ++--- .../llm-inference-service/actions.svelte | 30 +- .../kind-viewer-actions/node/actions.svelte | 28 +- .../utils/gpu-resources.svelte | 457 +++++++++--------- 4 files changed, 332 insertions(+), 313 deletions(-) diff --git a/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte b/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte index 219729d6..b932951b 100644 --- a/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte +++ b/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte @@ -2,12 +2,13 @@ import { ConnectError, createClient, type Transport } from '@connectrpc/connect'; import GpuIcon from '@lucide/svelte/icons/gpu'; import { ResourceService } from '@otterscale/api/resource/v1'; - import { PrometheusDriver } from 'prometheus-query'; import { getContext, type Snippet } from 'svelte'; + import { page } from '$app/state'; import * as Dialog from '$lib/components/ui/dialog'; import * as Item from '$lib/components/ui/item'; import { m } from '$lib/messages'; + import { clusterPrometheusDriver } from '$lib/prometheus'; import { fetchLLMInferenceServiceTopology, fetchNodeTopology } from './fetch-topology'; import { fetchGpuUtilization } from './fetch-utilization'; @@ -37,6 +38,12 @@ const name: string = $derived(object?.metadata?.name ?? ''); + /** + * GPU inventory is cluster scope (nodes, DCGM, HAMi): only viewers with cluster metrics + * access get the dialog at all, everyone else gets no trigger and sends no query. + */ + const canViewClusterMetrics = $derived(page.data.canViewClusterMetrics === true); + let open = $state(false); let isLoading = $state(false); let error = $state(null); @@ -51,14 +58,10 @@ * only costs the utilization bars — never the allocation view itself. */ async function attachUtilization(data: TopologyData, id: number) { - if (data.gpus.length === 0) return; + if (!canViewClusterMetrics || data.gpus.length === 0) return; try { - const prometheus = new PrometheusDriver({ - endpoint: `/proxy/${cluster}/prometheus`, - baseURL: '/api/v1', - headers: { 'x-proxy-target': 'api' } - }); + const prometheus = clusterPrometheusDriver(cluster); const utilization = await fetchGpuUtilization( prometheus, data.gpus.map((gpu) => gpu.device.id) @@ -75,6 +78,7 @@ } async function fetchData() { + if (!canViewClusterMetrics) return; const id = ++requestId; isLoading = true; error = null; @@ -107,61 +111,63 @@ ); - - - {#if trigger} - {@render trigger()} - {:else} - - - - - - {m.gpu_allocation()} - - - {/if} - - - - - {m.gpu_allocation_title({ name })} - - - {#if view === 'llm-inference-service'} - {m.gpu_allocation_model_description()} - {:else} - {m.gpu_allocation_node_description()} - {/if} - - - -
- {#if isLoading} -
- {m.gpu_allocation_loading()} -
- {:else if error} -
- {error} -
- {:else if isEmpty} -
+{#if canViewClusterMetrics} + + + {#if trigger} + {@render trigger()} + {:else} + + + + + + {m.gpu_allocation()} + + + {/if} + + + + + {m.gpu_allocation_title({ name })} + + {#if view === 'llm-inference-service'} - {m.gpu_allocation_empty_model()} + {m.gpu_allocation_model_description()} {:else} - {m.gpu_allocation_empty_node()} + {m.gpu_allocation_node_description()} {/if} -
- {:else if topologyData} - - {/if} -
- - + + + +
+ {#if isLoading} +
+ {m.gpu_allocation_loading()} +
+ {:else if error} +
+ {error} +
+ {:else if isEmpty} +
+ {#if view === 'llm-inference-service'} + {m.gpu_allocation_empty_model()} + {:else} + {m.gpu_allocation_empty_node()} + {/if} +
+ {:else if topologyData} + + {/if} +
+ + +{/if} diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte index df5768d9..e1dad7b7 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte @@ -4,6 +4,7 @@ import type { Schema } from '@sjsf/form'; import type { ValidateFunction } from 'ajv'; + import { page } from '$app/state'; import Delete from '$lib/components/kind-viewer/kind-viewer-actions/default/delete.svelte'; import Describe from '$lib/components/kind-viewer/kind-viewer-actions/default/describe.svelte'; import Edit from '$lib/components/kind-viewer/kind-viewer-actions/default/edit.svelte'; @@ -52,20 +53,23 @@ Inspect - { - e.preventDefault(); - }} - > - { - actionsOpen = false; + + {#if page.data.canViewClusterMetrics === true} + { + e.preventDefault(); }} - /> - + > + { + actionsOpen = false; + }} + /> + + {/if} { e.preventDefault(); diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte index 93e776e2..02f38d72 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/node/actions.svelte @@ -2,6 +2,7 @@ import Ellipsis from '@lucide/svelte/icons/ellipsis'; import type { Schema } from '@sjsf/form'; + import { page } from '$app/state'; import Describe from '$lib/components/kind-viewer/kind-viewer-actions/default/describe.svelte'; import View from '$lib/components/kind-viewer/kind-viewer-actions/default/view.svelte'; import Button from '$lib/components/ui/button/button.svelte'; @@ -46,19 +47,22 @@ - { - e.preventDefault(); - }} - > - { - actionsOpen = false; + + {#if page.data.canViewClusterMetrics === true} + { + e.preventDefault(); }} - /> - + > + { + actionsOpen = false; + }} + /> + + {/if} { e.preventDefault(); diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte index e00949ff..241a0065 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte @@ -2,7 +2,6 @@ import type { JsonValue } from '@bufbuild/protobuf'; import { GpuIcon } from '@lucide/svelte'; import lodash from 'lodash'; - import { PrometheusDriver } from 'prometheus-query'; import { onMount } from 'svelte'; import { page } from '$app/state'; @@ -11,250 +10,256 @@ import * as Modal from '$lib/components/ui/dialog'; import * as Item from '$lib/components/ui/item'; import * as Table from '$lib/components/ui/table'; + import { clusterPrometheusDriver } from '$lib/prometheus'; let gpus: Record[] = []; let gpusByModelName: Record[] = []; let gpusByHostName: Record[] = []; - const driver = new PrometheusDriver({ - endpoint: `/proxy/${page.params.cluster!}/prometheus`, - baseURL: '/api/v1', - headers: { 'x-proxy-target': 'api' } - }); - onMount(async () => { - const [fieldBufferResponse, vgpuLimitResponse, vgpuUsageResponse] = await Promise.all([ - driver.instantQuery( - '(DCGM_FI_DEV_FB_FREE + DCGM_FI_DEV_FB_RESERVED + DCGM_FI_DEV_FB_USED) * (1024 * 1024)' // DCGM return in Mi - ), - driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_limit_bytes)'), - driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_used_bytes)') - ]); + // DCGM and HAMi are cluster scope: a regular user would only get 403s. + if (page.data.canViewClusterMetrics !== true) return; + const driver = clusterPrometheusDriver(page.params.cluster!); - const vgpuLimits = vgpuLimitResponse.result.map((series) => { - return { ...series.metric.labels, limit: series.value?.value }; - }); - const vgpuUsages = vgpuUsageResponse.result.map((series) => { - return { ...series.metric.labels, usage: series.value?.value }; - }); + try { + const [fieldBufferResponse, vgpuLimitResponse, vgpuUsageResponse] = await Promise.all([ + driver.instantQuery( + '(DCGM_FI_DEV_FB_FREE + DCGM_FI_DEV_FB_RESERVED + DCGM_FI_DEV_FB_USED) * (1024 * 1024)' // DCGM return in Mi + ), + driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_limit_bytes)'), + driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_used_bytes)') + ]); - const limitsByDeviceUUID = lodash.mapValues( - lodash.groupBy(vgpuLimits, 'device_uuid'), - (group) => lodash.sumBy(group, 'limit') - ); - const usagesByDeviceUUID = lodash.mapValues( - lodash.groupBy(vgpuUsages, 'device_uuid'), - (group) => lodash.sumBy(group, 'usage') - ); + const vgpuLimits = vgpuLimitResponse.result.map((series) => { + return { ...series.metric.labels, limit: series.value?.value }; + }); + const vgpuUsages = vgpuUsageResponse.result.map((series) => { + return { ...series.metric.labels, usage: series.value?.value }; + }); - gpus = fieldBufferResponse.result.map((series) => { - const labels = series.metric.labels; - const uuid = lodash.get(labels, 'UUID'); - return { - ...labels, - total: lodash.get(series, ['value', 'value'], undefined), - limit: lodash.get(limitsByDeviceUUID, uuid, undefined), - usage: lodash.get(usagesByDeviceUUID, uuid, undefined) - }; - }); - gpusByModelName = Object.entries( - lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'modelName')) - ).map(([key, value]) => ({ - identifier: key, - limit: lodash.sumBy(value, 'limit'), - usage: lodash.sumBy(value, 'usage'), - total: lodash.sumBy(value, 'total') - })); - gpusByHostName = Object.entries(lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'hostname'))).map( - ([key, value]) => ({ + const limitsByDeviceUUID = lodash.mapValues( + lodash.groupBy(vgpuLimits, 'device_uuid'), + (group) => lodash.sumBy(group, 'limit') + ); + const usagesByDeviceUUID = lodash.mapValues( + lodash.groupBy(vgpuUsages, 'device_uuid'), + (group) => lodash.sumBy(group, 'usage') + ); + + gpus = fieldBufferResponse.result.map((series) => { + const labels = series.metric.labels; + const uuid = lodash.get(labels, 'UUID'); + return { + ...labels, + total: lodash.get(series, ['value', 'value'], undefined), + limit: lodash.get(limitsByDeviceUUID, uuid, undefined), + usage: lodash.get(usagesByDeviceUUID, uuid, undefined) + }; + }); + gpusByModelName = Object.entries( + lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'modelName')) + ).map(([key, value]) => ({ + identifier: key, + limit: lodash.sumBy(value, 'limit'), + usage: lodash.sumBy(value, 'usage'), + total: lodash.sumBy(value, 'total') + })); + gpusByHostName = Object.entries( + lodash.groupBy(gpus, (gpu) => lodash.get(gpu, 'hostname')) + ).map(([key, value]) => ({ identifier: key, limit: lodash.sumBy(value, 'limit'), usage: lodash.sumBy(value, 'usage'), total: lodash.sumBy(value, 'total') - }) - ); + })); + } catch (error) { + // The tables stay empty. + console.error('Failed to fetch GPU resources:', error); + } }); - - - - - e.preventDefault()} - > - - - GPU Resources - Amount of limits, usage, and total field buffers. - - {gpus.length} devices - +{#if page.data.canViewClusterMetrics === true} + + + + + e.preventDefault()} + > + + + GPU Resources + Amount of limits, usage, and total field buffers. + + {gpus.length} devices + - - - Summary - - {gpusByModelName.length} types - - - - - - - Identifier - usage - limit - total - - - - {#each gpusByModelName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + + + Summary + + {gpusByModelName.length} types + + + + + + + Identifier + usage + limit + total - {/each} - - + + + {#each gpusByModelName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + - - - Summary - - {gpusByHostName.length} hosts - - - - - - - Identifier - usage - limit - total - - - - {#each gpusByHostName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + + + Summary + + {gpusByHostName.length} hosts + + + + + + + Identifier + usage + limit + total - {/each} - - + + + {#each gpusByHostName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + - - - Details - - {gpus.length} devices - - - - - - - Model Name - Host Name - Device - UUID - usage - limit - total - - - - {#each gpus as gpu, index (index)} - - {gpu.modelName} - {gpu.hostname} - {gpu.device} - {gpu.UUID} - - {@const { value, unit } = - gpu.usage && typeof gpu.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpu.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.limit && typeof gpu.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpu.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.total && typeof gpu.total === 'number' - ? formatWithBinarySuffix(BigInt(gpu.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + + + Details + + {gpus.length} devices + + + + + + + Model Name + Host Name + Device + UUID + usage + limit + total - {/each} - - - - + + + {#each gpus as gpu, index (index)} + + {gpu.modelName} + {gpu.hostname} + {gpu.device} + {gpu.UUID} + + {@const { value, unit } = + gpu.usage && typeof gpu.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpu.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.limit && typeof gpu.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpu.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.total && typeof gpu.total === 'number' + ? formatWithBinarySuffix(BigInt(gpu.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + + + +{/if} From f9ac60e5bcc6167b3c7f1e8e49cb1ff4478a0ca2 Mon Sep 17 00:00:00 2001 From: hstsai Date: Thu, 8 Oct 2026 19:32:26 +0800 Subject: [PATCH 05/41] docs(dashboard): which metrics a namespace-scoped page may use Co-Authored-By: Claude Opus 5.5 --- README.md | 2 ++ docs/metrics.md | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 docs/metrics.md diff --git a/README.md b/README.md index 922e4014..6a358358 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ kubectl get secret -n otterscale-system otterscale-ca \ NODE_EXTRA_CA_CERTS=$PWD/otterscale-ca.crt pnpm dev --host ``` +- [Metrics in the dashboard](docs/metrics.md): which metrics a page may use, and why regular users see less. + ## Documentation Setup, configuration, and deployment guides will be published in the project documentation. See `.env.example` for the environment variables the app expects. diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 00000000..2dda3462 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,17 @@ +# Metrics in the dashboard + +The OtterScale server answers PromQL at two scopes: + +| Scope | Path | Who | What | +| --------- | ---------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------- | +| Cluster | `/proxy//prometheus` | cluster-admin, or bound to `otterscale:metrics-cluster-view` | any query | +| Namespace | `/proxy//prometheus-tenancy?namespace=` | anyone who may `get pods` in `` | any query, with `namespace=""` forced into every selector | + +Everyone else gets 403 at cluster scope, whatever the query. Create drivers with `clusterPrometheusDriver`, `namespacePrometheusDriver` or `workspacePrometheusDriver` (`src/lib/prometheus.ts`), never `new PrometheusDriver` with a hand-written endpoint. `page.data.canViewClusterMetrics` says which scope the viewer has. + +## Rules for namespace-scoped pages + +- Use only metrics whose `namespace` label is the tenant's namespace: kube-state-metrics, cAdvisor and kubelet volume metrics, vLLM, KubeVirt VMI metrics (`namespace`, not `exported_namespace`). +- Node, GPU (DCGM, HAMi), Ceph and KubeVirt component metrics are infrastructure: their `namespace` is the exporter's. A regular user can never see them, so hide the component when `canViewClusterMetrics` is false — do not show an empty chart or a 403. +- Do not select on `namespace` yourself with `=~`, `!=` or `!~` against other namespaces: the server refuses a selector that cannot match the enforced namespace with 400. +- Testing with an admin account proves nothing here: an admin keeps cluster scope. Run `e2e/tenant-metrics.test.ts` as a regular Workspace member (see the comment at its top). From 9f38c01583a758c78957c0aa13a07a64f14c00b1 Mon Sep 17 00:00:00 2001 From: hstsai Date: Mon, 12 Oct 2026 10:51:17 +0800 Subject: [PATCH 06/41] test(dashboard): e2e for a regular user's namespace-scoped metrics Against a deployed dashboard: no cluster or storage dashboards, every chart query goes to /prometheus-tenancy for their namespace only, the overview has data, and no infrastructure metric is asked for. Co-Authored-By: Claude Sonnet 5.5 --- e2e/tenant-metrics.test.ts | 164 +++++++++++++++++++++++++++++++++++++ playwright.config.ts | 23 ++++-- 2 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 e2e/tenant-metrics.test.ts diff --git a/e2e/tenant-metrics.test.ts b/e2e/tenant-metrics.test.ts new file mode 100644 index 00000000..60bc81ca --- /dev/null +++ b/e2e/tenant-metrics.test.ts @@ -0,0 +1,164 @@ +/* + * A regular user's dashboard: what they see, and that the charts carry data + * from their own namespace only. Runs against a deployed dashboard: + * + * E2E_BASE_URL=https:// E2E_IGNORE_HTTPS_ERRORS=1 \ + * E2E_TENANT_USERNAME=... E2E_TENANT_PASSWORD=... \ + * E2E_CLUSTER= E2E_WORKSPACE= E2E_NAMESPACE= \ + * pnpm exec playwright test e2e/tenant-metrics.test.ts + * + * The user must be a plain Workspace member (view is enough): not cluster-admin, + * not bound to otterscale:metrics-cluster-view, in no Keycloak group that + * grants either. An admin account proves nothing here — admins keep cluster + * scope. E2E_EXPECT_VM_DATA / E2E_EXPECT_MODEL_DATA=1 also require data on the + * compute / model pages (set them when the namespace runs a VM / a model). + */ +import { expect, type Page, test } from '@playwright/test'; + +const env = { + username: process.env.E2E_TENANT_USERNAME ?? '', + password: process.env.E2E_TENANT_PASSWORD ?? '', + cluster: process.env.E2E_CLUSTER ?? '', + workspace: process.env.E2E_WORKSPACE ?? '', + namespace: process.env.E2E_NAMESPACE ?? '' +}; + +test.skip( + !process.env.E2E_BASE_URL || Object.values(env).some((v) => !v), + 'needs E2E_BASE_URL and the E2E_TENANT_* / E2E_CLUSTER / E2E_WORKSPACE / E2E_NAMESPACE variables' +); + +type Sample = { + metric?: Record; + value?: [number, string]; + values?: [number, string][]; +}; +type Captured = { url: URL; status: number; result: Sample[] }; + +/** Infrastructure metrics a namespace-scoped page must not even ask for. */ +const INFRASTRUCTURE = [ + 'ceph_cluster_total_bytes', + 'kubevirt_info', + 'kubevirt_virt_controller_ready_status', + 'DCGM_FI_DEV_FB_USED', + 'DCGM_FI_DEV_XID_ERRORS' +]; + +function watchMetrics(page: Page) { + const tenancy: Captured[] = []; + const clusterScope: string[] = []; + page.on('request', (req) => { + if (req.url().includes(`/proxy/${env.cluster}/prometheus/`)) clusterScope.push(req.url()); + }); + page.on('response', async (res) => { + const url = new URL(res.url()); + if (!url.pathname.startsWith(`/proxy/${env.cluster}/prometheus-tenancy/`)) return; + let result: Sample[] = []; + try { + const body = await res.json(); + result = Array.isArray(body?.data?.result) ? body.data.result : []; + } catch { + // metadata and label answers are not vectors + } + tenancy.push({ url, status: res.status(), result }); + }); + return { tenancy, clusterScope }; +} + +const sentQueries = (captured: Captured[]) => + captured.map((c) => + [c.url.searchParams.get('query') ?? '', ...c.url.searchParams.getAll('match[]')].join(' ') + ); + +function hasData(captured: Captured[]) { + return captured.some((c) => + c.result.some((s) => { + const v = Number(s.value?.[1] ?? s.values?.at(-1)?.[1]); + return Number.isFinite(v) && v !== 0; + }) + ); +} + +function expectOwnNamespaceOnly(captured: Captured[]) { + expect(captured.length, 'no namespace-scoped query was sent').toBeGreaterThan(0); + for (const c of captured) { + expect(c.status, `${c.url.pathname} answered ${c.status}`).toBeLessThan(400); + expect(c.url.searchParams.getAll('namespace')).toEqual([env.namespace]); + for (const s of c.result) { + if (s.metric?.namespace !== undefined) expect(s.metric.namespace).toBe(env.namespace); + } + } +} + +async function login(page: Page) { + await page.goto('/login'); + // The otterscale realm asks for the username first, then the password on a second page. + await page.locator('#username').fill(env.username); + await page.locator('#kc-login').click(); + await page.locator('#password').fill(env.password); + await page.locator('#kc-login').click(); + // Back on the dashboard (not still on Keycloak or its /login/callback). + await page.waitForURL( + (url) => + url.origin === new URL(process.env.E2E_BASE_URL ?? '').origin && + !url.pathname.startsWith('/login') + ); +} + +const dashboard = (name: string) => `/${env.cluster}/${env.workspace}/dashboard/${name}`; + +async function visit(page: Page, name: string) { + const watched = watchMetrics(page); + await page.goto(dashboard(name)); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(3_000); // widgets query after mount + return watched; +} + +test.beforeEach(async ({ page }) => { + await login(page); +}); + +test('the sidebar has no cluster or storage dashboard', async ({ page }) => { + await page.goto(dashboard('overview')); + await page.waitForLoadState('networkidle'); + await expect(page.locator(`a[href$="/dashboard/cluster"]`)).toHaveCount(0); + await expect(page.locator(`a[href$="/dashboard/storage"]`)).toHaveCount(0); + await expect(page.locator(`a[href$="/dashboard/model"]`).first()).toBeVisible(); +}); + +for (const name of ['cluster', 'storage']) { + test(`opening the ${name} dashboard directly lands on the overview`, async ({ page }) => { + await page.goto(dashboard(name)); + await page.waitForURL((url) => url.pathname.endsWith('/dashboard/overview')); + }); +} + +test('overview shows their namespace, with data, and nothing cluster-wide', async ({ page }) => { + const { tenancy, clusterScope } = await visit(page, 'overview'); + expectOwnNamespaceOnly(tenancy); + expect(hasData(tenancy), 'overview charts are empty').toBe(true); + expect(clusterScope).toEqual([]); + for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); +}); + +test('compute stays in their namespace and hides KubeVirt components', async ({ page }) => { + const { tenancy, clusterScope } = await visit(page, 'compute'); + expectOwnNamespaceOnly(tenancy); + expect(clusterScope).toEqual([]); + for (const q of sentQueries(tenancy)) { + for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); + expect(q).not.toContain('exported_namespace'); + } + if (process.env.E2E_EXPECT_VM_DATA === '1') + expect(hasData(tenancy), 'compute charts are empty').toBe(true); +}); + +test('model stays in their namespace and hides GPU memory and XID', async ({ page }) => { + const { tenancy, clusterScope } = await visit(page, 'model'); + expectOwnNamespaceOnly(tenancy); + expect(clusterScope).toEqual([]); + for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); + if (process.env.E2E_EXPECT_MODEL_DATA === '1') + expect(hasData(tenancy), 'model charts are empty').toBe(true); +}); diff --git a/playwright.config.ts b/playwright.config.ts index f6c81af8..f7a28689 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,9 +1,22 @@ import { defineConfig } from '@playwright/test'; +// E2E_BASE_URL runs the suite against a deployed dashboard (e2e/tenant-metrics +// needs a real Keycloak user and hub); without it, against a local preview. +const baseURL = process.env.E2E_BASE_URL; + export default defineConfig({ - webServer: { - command: 'npm run build && npm run preview', - port: 4173 - }, - testDir: 'e2e' + testDir: 'e2e', + ...(baseURL + ? { + use: { + baseURL, + ignoreHTTPSErrors: process.env.E2E_IGNORE_HTTPS_ERRORS === '1' + } + } + : { + webServer: { + command: 'npm run build && npm run preview', + port: 4173 + } + }) }); From bd718bb08807640de1a49dd87626bcb6333e3ba3 Mon Sep 17 00:00:00 2001 From: hstsai Date: Mon, 12 Oct 2026 10:53:42 +0800 Subject: [PATCH 07/41] test(dashboard): wait out Keycloak under /auth/ before the e2e login returns The lab serves Keycloak on the dashboard's origin under /auth/, so an origin check alone resolved while still on the password page and the next navigation could abort the login callback. Co-Authored-By: Claude Sonnet 5.5 --- e2e/tenant-metrics.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/tenant-metrics.test.ts b/e2e/tenant-metrics.test.ts index 60bc81ca..5fda125f 100644 --- a/e2e/tenant-metrics.test.ts +++ b/e2e/tenant-metrics.test.ts @@ -97,11 +97,12 @@ async function login(page: Page) { await page.locator('#kc-login').click(); await page.locator('#password').fill(env.password); await page.locator('#kc-login').click(); - // Back on the dashboard (not still on Keycloak or its /login/callback). + // Back on the dashboard: not on another origin's Keycloak, nor on one mounted under /auth/, + // nor still on /login/callback. await page.waitForURL( (url) => url.origin === new URL(process.env.E2E_BASE_URL ?? '').origin && - !url.pathname.startsWith('/login') + !/^\/(auth|login)(\/|$)/.test(url.pathname) ); } From 44c5fa66cf8e9e4cf9bced5a3198cfd04aace9f7 Mon Sep 17 00:00:00 2001 From: hstsai Date: Mon, 12 Oct 2026 11:21:25 +0800 Subject: [PATCH 08/41] refactor(dashboard): every cluster-scope driver from clusterPrometheusDriver The cluster and storage dashboards and the workspaces viewer still built their PrometheusDriver by hand, against the rule docs/metrics.md states. Co-Authored-By: Claude Opus 5.5 --- src/lib/components/dashboard/cluster/index.svelte | 11 +++-------- src/lib/components/dashboard/storage/index.svelte | 11 +++-------- .../resource-viewer/viewers/workspaces-viewer.svelte | 9 +++------ 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/src/lib/components/dashboard/cluster/index.svelte b/src/lib/components/dashboard/cluster/index.svelte index 76b603bb..fa35bfb4 100644 --- a/src/lib/components/dashboard/cluster/index.svelte +++ b/src/lib/components/dashboard/cluster/index.svelte @@ -1,6 +1,6 @@ - -{#if !isLoaded} - - - {#each Array(13).keys() as index (index)} - {#if index % 2 === 0} - {#if index % 3 !== 0} - {#if index % 5 === 0} - {#if index % 7 !== 0} - {#if index % 11 === 0} - - {:else} - - {/if} - {:else} - - {/if} - {:else} - - {/if} - {:else} - - {/if} - {:else} - - {/if} - {/each} - - -{:else} - {#snippet memberGrid()} -
- {#each filteredMembers as member (member.subject)} - - - - {getMemberInitials(member)} - - - - - {getMemberLabel(member)} - - {#if member.serviceAccount} - service account - {/if} - {member.role} - - - - - - {#if member.username} - {member.username} - {/if} - {member.subject} - - - {#if member.username} -

username: {member.username}

- {/if} -

subject: {member.subject}

- - - - - - {/each} -
- {/snippet} - - - - - - {@const conditions = object.status?.conditions ?? []} - {@const readyCondition = conditions.find((condition) => condition.type === 'Ready')} - {@const isReady = readyCondition?.status === 'True' ? true : false} - - - - - - - - Status - - - - - - {#if conditions.length > 0} - {#if isReady} - - - - {readyCondition?.reason} - - {readyCondition?.message} - - - {:else} - {#each [...conditions].sort((p, n) => { - const previous = new Date(p.lastTransitionTime ?? 0).getTime(); - const next = new Date(n.lastTransitionTime ?? 0).getTime(); - return next - previous; - }) as condition, index (index)} - - - - {condition?.reason} - - {condition?.message} - - - {/each} - {/if} - {:else} - - - - - - No Status Available - - There are no status conditions to display for this workspace. - - - - {/if} - - - - - - {@const resourceQuotaHard = object.spec?.resourceQuota?.hard ?? {}} - - - - - - - - Resource Quota - - - - - {#snippet child({ props })} - { - isResourceQuotasGrid = !isResourceQuotasGrid; - }} - class="data-[state=on]:*:[svg]:fill-muted-foreground/50" - > - - - {/snippet} - - Toggle Grid Layout - - - - - - - {#if Object.keys(resourceQuotaHard).length > 0} -
- {#each Object.keys(resourceQuotaHard) as key, index (index)} - - - - {key} - - - {resourceQuotaUsed[key] !== undefined - ? resourceQuotaUsed[key] - : '?'}/{formatHardValue(key, resourceQuotaHard[key])} - - - - {/each} -
- {:else} - - - - - - No Resource Quota Configured - - Resource Quota is not configured yet. Please click the edit button at the top - right to configure Resource Quota. - - - - {/if} - - - - - - {@const limits = object.spec?.limitRange?.limits ?? []} - - - - - - - - Limit Range - - - - - - {#if limits.length > 0} - {#each limits as limit, index (index)} - {@const { type, ...thresholds } = limit} - - - - {type} - - - - {#each Object.entries(thresholds) as [key, values], index (index)} - {#if values && typeof values === 'object'} - - - - {key} - - - {#each Object.entries(values) as [key, value], index (index)} -

{key}:{value}

- {/each} - - - - {/if} - {/each} - - - - {/each} - {:else} - - - - - - No Limit Range Configured - - Limit Range is not configured yet. Please click the edit button at the top right - to configure Limit Range. - - - - {/if} - - - - - - - - - - - - - - Network Isolation - - - - - - - - - Enabled - - {@const enabled = object.spec?.networkIsolation?.enabled ?? null} - {#if enabled === true} - - {:else} - - {/if} - - - - - {@const allowedNamespaces = object.spec?.networkIsolation?.allowedNamespaces ?? []} - - - - Allowed Namespaces - - - {#if allowedNamespaces.length > 0} -
- {#each allowedNamespaces as allowedNamespace, index (index)} - - - {allowedNamespace} - - {/each} -
- {:else} - - -

No namespaces allowed

- - {/if} - - - - {allowedNamespaces.length} - - - - - - - - - - - - - - - - Members - {#if memberRoleSummary} - {memberRoleSummary} - {/if} - - - - -
- {#if members.length > MEMBERS_SCROLL_THRESHOLD} - - - - - {#snippet child({ props })} - - - - {/snippet} - - - - Filter by Role - - {#each MEMBER_ROLE_FILTERS as roleFilter (roleFilter)} - - {roleFilter} - - {roleFilter === 'all' ? members.length : memberRoleCounts[roleFilter]} - - - {/each} - - - - Filter by Role - - {/if} - - - {filteredMembers.length} - -
- - - - {#if members.length === 0} - - - - - - No Members - - No members have been granted access to this workspace yet. - - - - {:else if filteredMembers.length === 0} - - - - - - No Matching Members - - No member holds the {memberRoleFilter} role in this workspace. - - - - {:else if filteredMembers.length > MEMBERS_SCROLL_THRESHOLD} - -
- {@render memberGrid()} -
- - {:else} - {@render memberGrid()} - {/if} - - - - - {@const namespace = object.status?.namespaceRef?.name ?? ''} - - {#if relatedResources.length > 0} - - - - Related Resources - - {relatedResources.length} resources related to {object.kind} - {object.metadata?.name} - - - - -
- {#each relatedResources as relatedResource (relatedResource.kind + relatedResource.name)} - {#if !relatedResource.namespaced || (relatedResource.namespaced && namespace)} - - {#snippet child({ props })} - {@const urlSearchParameters = new URLSearchParams({ - group: relatedResource.group, - version: relatedResource.version, - kind: relatedResource.kind, - resource: relatedResource.resource, - ...(relatedResource.namespaced ? { namespace: namespace } : {}) - })} - {@const url = resolve( - `/(auth)/${page.params.cluster}/${page.params.workspace}/${relatedResource.name}?${urlSearchParameters}` - )} - - - {relatedResource.name} - - {relatedResource.resource}.{relatedResource.group - ? `${relatedResource.group}/${relatedResource.version}` - : relatedResource.version} - - - - - - - {/snippet} - - {/if} - {/each} -
- {:else} - - - - - - No Related Resources - - There are no related resources to display for this workspace. - - - - {/if} - - -{/if} From fe4a39ee68f7f42921584a397d779d38fb578e70 Mon Sep 17 00:00:00 2001 From: hstsai Date: Tue, 13 Oct 2026 20:58:57 +0800 Subject: [PATCH 20/41] fix(dashboard): compute's all-namespaces choice selects every namespace - namespace="" matched only series without the label - so an admin choosing all namespaces saw nothing --- .../dashboard/compute/analytics/vm-picker.svelte | 3 ++- .../components/dashboard/compute/overview/cpu.svelte | 8 ++++---- .../dashboard/compute/overview/instance.svelte | 3 ++- .../dashboard/compute/overview/memory.svelte | 4 ++-- .../compute/overview/network-traffic.svelte | 6 +++--- .../dashboard/compute/overview/throughput.svelte | 6 +++--- .../compute/overview/virtual-machine.svelte | 3 ++- src/lib/components/dashboard/namespace-scope.spec.ts | 6 ++++++ src/lib/prometheus.spec.ts | 12 ++++++++++++ src/lib/prometheus.ts | 9 +++++++++ 10 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/lib/components/dashboard/compute/analytics/vm-picker.svelte b/src/lib/components/dashboard/compute/analytics/vm-picker.svelte index 9898b3f0..ebc311c0 100644 --- a/src/lib/components/dashboard/compute/analytics/vm-picker.svelte +++ b/src/lib/components/dashboard/compute/analytics/vm-picker.svelte @@ -16,6 +16,7 @@ type ComboboxEnumeration } from '$lib/components/dynamic-form/widgets/combobox.svelte'; import { m } from '$lib/messages'; + import { namespaceSelector } from '$lib/prometheus'; let { prometheusDriver, @@ -74,7 +75,7 @@ isLoaded = false; try { const response = await prometheusDriver.instantQuery( - `kubevirt_vmi_info{` + (ns ? `namespace="${ns}"` : '') + `}` + `kubevirt_vmi_info${namespaceSelector(ns ?? '')}` ); if (cancelled) return; const series = (response.result ?? []) as { diff --git a/src/lib/components/dashboard/compute/overview/cpu.svelte b/src/lib/components/dashboard/compute/overview/cpu.svelte index 1f2d8df5..3ef995c2 100644 --- a/src/lib/components/dashboard/compute/overview/cpu.svelte +++ b/src/lib/components/dashboard/compute/overview/cpu.svelte @@ -11,7 +11,7 @@ import * as Card from '$lib/components/ui/card'; import * as Chart from '$lib/components/ui/chart/index.js'; import { m } from '$lib/messages'; - import { computeStep } from '$lib/prometheus'; + import { computeStep, namespaceSelector } from '$lib/prometheus'; let { prometheusDriver, @@ -46,7 +46,7 @@ const rangeEnd = new Date(endMs); const step = computeStep(start.getTime(), endMs); const response = await prometheusDriver.rangeQuery( - `avg(rate(kubevirt_vmi_cpu_usage_seconds_total{namespace="${namespace}"}[5m]))`, + `avg(rate(kubevirt_vmi_cpu_usage_seconds_total${namespaceSelector(namespace)}[5m]))`, start, rangeEnd, step @@ -57,7 +57,7 @@ let cpuWait: SampleValue | undefined = $state(undefined); async function fetchCPUWait() { const response = await prometheusDriver.instantQuery( - `avg(rate(kubevirt_vmi_vcpu_wait_seconds_total{namespace="${namespace}"}[5m]))` + `avg(rate(kubevirt_vmi_vcpu_wait_seconds_total${namespaceSelector(namespace)}[5m]))` ); cpuWait = response.result[0]?.value ?? undefined; } @@ -65,7 +65,7 @@ let cpuDelay: SampleValue | undefined = $state(undefined); async function fetchCPUDelay() { const response = await prometheusDriver.instantQuery( - `avg(rate(kubevirt_vmi_vcpu_delay_seconds_total{namespace="${namespace}"}[5m]))` + `avg(rate(kubevirt_vmi_vcpu_delay_seconds_total${namespaceSelector(namespace)}[5m]))` ); cpuDelay = response.result[0]?.value ?? undefined; } diff --git a/src/lib/components/dashboard/compute/overview/instance.svelte b/src/lib/components/dashboard/compute/overview/instance.svelte index 39368894..f50012af 100644 --- a/src/lib/components/dashboard/compute/overview/instance.svelte +++ b/src/lib/components/dashboard/compute/overview/instance.svelte @@ -9,6 +9,7 @@ import { ReloadManager } from '$lib/components/custom/reloader'; import * as Card from '$lib/components/ui/card'; import { m } from '$lib/messages'; + import { namespaceSelector } from '$lib/prometheus'; let { prometheusDriver, @@ -19,7 +20,7 @@ let instances: SampleValue | undefined = $state(undefined); async function fetchInstances() { const response = await prometheusDriver.instantQuery( - `count(kubevirt_vmi_info{namespace="${namespace}"})` + `count(kubevirt_vmi_info${namespaceSelector(namespace)})` ); instances = response.result[0]?.value ?? undefined; } diff --git a/src/lib/components/dashboard/compute/overview/memory.svelte b/src/lib/components/dashboard/compute/overview/memory.svelte index 058c6995..b2801402 100644 --- a/src/lib/components/dashboard/compute/overview/memory.svelte +++ b/src/lib/components/dashboard/compute/overview/memory.svelte @@ -11,7 +11,7 @@ import * as Card from '$lib/components/ui/card'; import * as Chart from '$lib/components/ui/chart/index.js'; import { m } from '$lib/messages'; - import { computeStep } from '$lib/prometheus'; + import { computeStep, namespaceSelector } from '$lib/prometheus'; let { prometheusDriver, @@ -46,7 +46,7 @@ const rangeEnd = new Date(endMs); const step = computeStep(start.getTime(), endMs); const response = await prometheusDriver.rangeQuery( - `avg(kubevirt_vmi_memory_resident_bytes{namespace="${namespace}"}) / avg(kubevirt_vmi_memory_domain_bytes{namespace="${namespace}"})`, + `avg(kubevirt_vmi_memory_resident_bytes${namespaceSelector(namespace)}) / avg(kubevirt_vmi_memory_domain_bytes${namespaceSelector(namespace)})`, start, rangeEnd, step diff --git a/src/lib/components/dashboard/compute/overview/network-traffic.svelte b/src/lib/components/dashboard/compute/overview/network-traffic.svelte index a80be088..f8c6abe7 100644 --- a/src/lib/components/dashboard/compute/overview/network-traffic.svelte +++ b/src/lib/components/dashboard/compute/overview/network-traffic.svelte @@ -12,7 +12,7 @@ import * as Chart from '$lib/components/ui/chart/index.js'; import { formatIO } from '$lib/formatter'; import { m } from '$lib/messages'; - import { computeStep, fetchMultipleFlattenedRange } from '$lib/prometheus'; + import { computeStep, fetchMultipleFlattenedRange, namespaceSelector } from '$lib/prometheus'; let { prometheusDriver, @@ -53,8 +53,8 @@ const points = await fetchMultipleFlattenedRange( prometheusDriver, { - receive: `avg(rate(kubevirt_vmi_network_receive_bytes_total{namespace="${namespace}"}[5m]))`, - transmit: `avg(rate(kubevirt_vmi_network_transmit_bytes_total{namespace="${namespace}"}[5m]))` + receive: `avg(rate(kubevirt_vmi_network_receive_bytes_total${namespaceSelector(namespace)}[5m]))`, + transmit: `avg(rate(kubevirt_vmi_network_transmit_bytes_total${namespaceSelector(namespace)}[5m]))` }, start, rangeEnd, diff --git a/src/lib/components/dashboard/compute/overview/throughput.svelte b/src/lib/components/dashboard/compute/overview/throughput.svelte index ddb367cb..cc501121 100644 --- a/src/lib/components/dashboard/compute/overview/throughput.svelte +++ b/src/lib/components/dashboard/compute/overview/throughput.svelte @@ -12,7 +12,7 @@ import * as Chart from '$lib/components/ui/chart/index.js'; import { formatIO } from '$lib/formatter'; import { m } from '$lib/messages'; - import { computeStep, fetchMultipleFlattenedRange } from '$lib/prometheus'; + import { computeStep, fetchMultipleFlattenedRange, namespaceSelector } from '$lib/prometheus'; let { prometheusDriver, @@ -53,8 +53,8 @@ const points = await fetchMultipleFlattenedRange( prometheusDriver, { - read: `avg(rate(kubevirt_vmi_storage_read_traffic_bytes_total{namespace="${namespace}"}[5m]))`, - write: `avg(rate(kubevirt_vmi_storage_write_traffic_bytes_total{namespace="${namespace}"}[5m]))` + read: `avg(rate(kubevirt_vmi_storage_read_traffic_bytes_total${namespaceSelector(namespace)}[5m]))`, + write: `avg(rate(kubevirt_vmi_storage_write_traffic_bytes_total${namespaceSelector(namespace)}[5m]))` }, start, rangeEnd, diff --git a/src/lib/components/dashboard/compute/overview/virtual-machine.svelte b/src/lib/components/dashboard/compute/overview/virtual-machine.svelte index 4287e308..02138795 100644 --- a/src/lib/components/dashboard/compute/overview/virtual-machine.svelte +++ b/src/lib/components/dashboard/compute/overview/virtual-machine.svelte @@ -9,6 +9,7 @@ import { ReloadManager } from '$lib/components/custom/reloader'; import * as Card from '$lib/components/ui/card'; import { m } from '$lib/messages'; + import { namespaceSelector } from '$lib/prometheus'; let { prometheusDriver, @@ -19,7 +20,7 @@ let virtualMachines: SampleValue | undefined = $state(undefined); async function fetchVirtualMachines() { const response = await prometheusDriver.instantQuery( - `count(kubevirt_vm_starting_status_last_transition_timestamp_seconds{namespace="${namespace}"})` + `count(kubevirt_vm_starting_status_last_transition_timestamp_seconds${namespaceSelector(namespace)})` ); virtualMachines = response.result[0]?.value ?? undefined; } diff --git a/src/lib/components/dashboard/namespace-scope.spec.ts b/src/lib/components/dashboard/namespace-scope.spec.ts index 8cafaaed..e06f4d32 100644 --- a/src/lib/components/dashboard/namespace-scope.spec.ts +++ b/src/lib/components/dashboard/namespace-scope.spec.ts @@ -21,4 +21,10 @@ describe('compute dashboard', () => { ); expect(offenders).toEqual([]); }); + + it('builds namespace matchers with namespaceSelector, so "all namespaces" selects all', () => { + const dir = join(process.cwd(), 'src/lib/components/dashboard/compute'); + const offenders = files(dir).filter((f) => /namespace="\$\{/.test(readFileSync(f, 'utf8'))); + expect(offenders).toEqual([]); + }); }); diff --git a/src/lib/prometheus.spec.ts b/src/lib/prometheus.spec.ts index 176f2bea..d8630389 100644 --- a/src/lib/prometheus.spec.ts +++ b/src/lib/prometheus.spec.ts @@ -8,6 +8,7 @@ import { clusterPrometheusDriver, dashboardBody, namespacePrometheusDriver, + namespaceSelector, workspacePrometheusDriver } from './prometheus'; @@ -132,3 +133,14 @@ describe('dashboardBody', () => { expect(dashboardBody(base)).toBe('empty'); }); }); + +describe('namespaceSelector', () => { + it('selects one namespace, escaped', () => { + expect(namespaceSelector('ws-a')).toBe('{namespace="ws-a"}'); + expect(namespaceSelector('a"b')).toBe('{namespace="a\\"b"}'); + }); + + it('selects every namespace for the empty choice', () => { + expect(namespaceSelector('')).toBe(''); + }); +}); diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 4eaa49d9..0eb50fb0 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -93,6 +93,15 @@ export function escapePromqlStringLiteral(value: string): string { return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n'); } +/** + * `{namespace=""}`, or nothing for an admin's "all namespaces" (the empty + * choice): namespace="" would match only series without the label. At + * namespace scope the server pins the tenant's namespace either way. + */ +export function namespaceSelector(namespace: string): string { + return namespace ? `{namespace="${escapePromqlStringLiteral(namespace)}"}` : ''; +} + /** * Restrict a per-pod KSM selector to non-terminated pods — the set `kubectl describe node` * counts. KSM keeps emitting requests/limits for Succeeded/Failed pods until the object is From 6af47e9a0a088ba5312f939f106fd15803825e99 Mon Sep 17 00:00:00 2001 From: hstsai Date: Tue, 13 Oct 2026 21:25:06 +0800 Subject: [PATCH 21/41] docs(dashboard): say the namespace rewrite is prom-label-proxy's - the server rewrites selectors the way prom-label-proxy does it - docs and the prometheus.ts header no longer say it is forced - metrics.md points optional namespace filters at namespaceSelector --- docs/metrics.md | 9 +++++---- src/lib/prometheus.ts | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 2dda3462..01667fc0 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -2,10 +2,10 @@ The OtterScale server answers PromQL at two scopes: -| Scope | Path | Who | What | -| --------- | ---------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------- | -| Cluster | `/proxy//prometheus` | cluster-admin, or bound to `otterscale:metrics-cluster-view` | any query | -| Namespace | `/proxy//prometheus-tenancy?namespace=` | anyone who may `get pods` in `` | any query, with `namespace=""` forced into every selector | +| Scope | Path | Who | What | +| --------- | ---------------------------------------------------- | ------------------------------------------------------------ | --------------------------------------------------------------------------- | +| Cluster | `/proxy//prometheus` | cluster-admin, or bound to `otterscale:metrics-cluster-view` | any query | +| Namespace | `/proxy//prometheus-tenancy?namespace=` | anyone who may `get pods` in `` | any query, rewritten to `namespace=""` the way prom-label-proxy does it | Everyone else gets 403 at cluster scope, whatever the query. Create drivers with `clusterPrometheusDriver`, `namespacePrometheusDriver` or `workspacePrometheusDriver` (`src/lib/prometheus.ts`), never `new PrometheusDriver` with a hand-written endpoint. `page.data.canViewClusterMetrics` says which scope the viewer has. @@ -14,4 +14,5 @@ Everyone else gets 403 at cluster scope, whatever the query. Create drivers with - Use only metrics whose `namespace` label is the tenant's namespace: kube-state-metrics, cAdvisor and kubelet volume metrics, vLLM, KubeVirt VMI metrics (`namespace`, not `exported_namespace`). - Node, GPU (DCGM, HAMi), Ceph and KubeVirt component metrics are infrastructure: their `namespace` is the exporter's. A regular user can never see them, so hide the component when `canViewClusterMetrics` is false — do not show an empty chart or a 403. - Do not select on `namespace` yourself with `=~`, `!=` or `!~` against other namespaces: the server refuses a selector that cannot match the enforced namespace with 400. +- Build an optional namespace filter with `namespaceSelector(ns)` (`src/lib/prometheus.ts`): it escapes the value and leaves the matcher out for an admin's empty "all namespaces" choice, where `namespace=""` would match only series without the label. - Testing with an admin account proves nothing here: an admin keeps cluster scope. Run `e2e/tenant-metrics.test.ts` as a regular Workspace member (see the comment at its top). diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 0eb50fb0..59f01eb8 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -11,9 +11,10 @@ import { m } from '$lib/messages'; * otterscale:metrics-cluster-view); everyone else gets 403 whatever they ask. * * Namespace scope (`/proxy//prometheus-tenancy?namespace=`) takes - * any query too, from anyone who may get pods in , with namespace="" - * forced into every selector. A namespace-scoped page therefore only works - * with metrics whose `namespace` label is the tenant's — see docs/metrics.md. + * any query too, from anyone who may get pods in , rewritten by the server + * to namespace="" the way prom-label-proxy does it. A namespace-scoped page + * therefore only works with metrics whose `namespace` label is the tenant's — + * see docs/metrics.md. */ const PROXY_HEADERS = { 'x-proxy-target': 'api' }; From 8c6b467c92f481e24891f956a7bc6a423c300303 Mon Sep 17 00:00:00 2001 From: hstsai Date: Tue, 13 Oct 2026 21:25:06 +0800 Subject: [PATCH 22/41] fix(dashboard): say the unknown-access banner hides the menu entries - cluster and storage are hidden from the menu; a bookmark still opens - the retry spec also covers a thrown error followed by an answer --- messages/en.json | 2 +- messages/zh-hant.json | 2 +- src/lib/server/metrics-access.spec.ts | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/messages/en.json b/messages/en.json index fab88765..1f53c73a 100644 --- a/messages/en.json +++ b/messages/en.json @@ -410,7 +410,7 @@ "wait": "Wait", "delay": "Delay", "no_data_display": "No data to display", - "cluster_metrics_access_unknown": "Couldn't confirm your cluster metrics access, so the Cluster and Storage dashboards are hidden for now. Refresh the page to try again.", + "cluster_metrics_access_unknown": "Couldn't confirm your cluster metrics access, so the Cluster and Storage dashboards are hidden from the menu for now. Refresh the page to try again.", "cluster_metrics_required": "The Cluster and Storage dashboards need cluster metrics access. Ask your administrator for it.", "workspace_namespace_unknown": "Couldn't find this workspace's namespace, so there are no metrics to show. Refresh the page, or ask your administrator to check the workspace.", "no_traffic_display": "No requests in this range", diff --git a/messages/zh-hant.json b/messages/zh-hant.json index f7f84452..5b72931a 100644 --- a/messages/zh-hant.json +++ b/messages/zh-hant.json @@ -410,7 +410,7 @@ "wait": "等待", "delay": "延遲", "no_data_display": "尚無資料可顯示", - "cluster_metrics_access_unknown": "暫時無法確認叢集指標權限，叢集與儲存儀表板先隱藏。請重新整理頁面再試一次。", + "cluster_metrics_access_unknown": "暫時無法確認叢集指標權限，叢集與儲存儀表板先從選單隱藏。請重新整理頁面再試一次。", "cluster_metrics_required": "叢集與儲存儀表板需要叢集指標權限，請洽管理員。", "workspace_namespace_unknown": "找不到這個工作區的命名空間，沒有可以顯示的指標。請重新整理頁面，或請管理員檢查工作區設定。", "no_traffic_display": "此區間內沒有請求", diff --git a/src/lib/server/metrics-access.spec.ts b/src/lib/server/metrics-access.spec.ts index 601cc839..165fcc6c 100644 --- a/src/lib/server/metrics-access.spec.ts +++ b/src/lib/server/metrics-access.spec.ts @@ -72,6 +72,10 @@ describe('checkClusterMetricsAccess', () => { const f = fakeFetch(status(503), allowed(true)); await expect(checkClusterMetricsAccess(f, 'c1')).resolves.toBe('allowed'); expect(f).toHaveBeenCalledTimes(2); + + const thrown = fakeFetch(() => new Error('blip'), allowed(true)); + await expect(checkClusterMetricsAccess(thrown, 'c1')).resolves.toBe('allowed'); + expect(thrown).toHaveBeenCalledTimes(2); expect(log).not.toHaveBeenCalled(); } finally { log.mockRestore(); From cded6ec8c74f6cd8d97bddb8ca0fc2b941e655a8 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 14:01:42 +0800 Subject: [PATCH 23/41] feat(dashboard): add a client for the server's named shared queries - sends a query name and parameters, never promql - namespace is the workspace's, left out when it has none - returns prometheus-query results, so callers keep their types --- docs/metrics.md | 13 +++--- src/lib/prometheus.spec.ts | 59 ++++++++++++++++++++++++ src/lib/prometheus.ts | 92 +++++++++++++++++++++++++++++++++++++- 3 files changed, 157 insertions(+), 7 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 01667fc0..84299c5c 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -2,17 +2,18 @@ The OtterScale server answers PromQL at two scopes: -| Scope | Path | Who | What | -| --------- | ---------------------------------------------------- | ------------------------------------------------------------ | --------------------------------------------------------------------------- | -| Cluster | `/proxy//prometheus` | cluster-admin, or bound to `otterscale:metrics-cluster-view` | any query | -| Namespace | `/proxy//prometheus-tenancy?namespace=` | anyone who may `get pods` in `` | any query, rewritten to `namespace=""` the way prom-label-proxy does it | +| Scope | Path | Who | What | +| --------- | ---------------------------------------------------- | ------------------------------------------------------------ | -------------------------------------------------------------------------------- | +| Cluster | `/proxy//prometheus` | cluster-admin, or bound to `otterscale:metrics-cluster-view` | any query | +| Namespace | `/proxy//prometheus-tenancy?namespace=` | anyone who may `get pods` in `` | any query, rewritten to `namespace=""` the way prom-label-proxy does it | +| Shared | `/proxy//prometheus-shared/` | the namespace scope's callers, and cluster scope | one of the server's named queries: per GPU card, per host, or a total; no PromQL | -Everyone else gets 403 at cluster scope, whatever the query. Create drivers with `clusterPrometheusDriver`, `namespacePrometheusDriver` or `workspacePrometheusDriver` (`src/lib/prometheus.ts`), never `new PrometheusDriver` with a hand-written endpoint. `page.data.canViewClusterMetrics` says which scope the viewer has. +Everyone else gets 403 at cluster scope, whatever the query. Create drivers with `clusterPrometheusDriver`, `namespacePrometheusDriver` or `workspacePrometheusDriver` (`src/lib/prometheus.ts`), never `new PrometheusDriver` with a hand-written endpoint. `page.data.canViewClusterMetrics` says which scope the viewer has. Use `sharedMetrics(cluster, namespace)` for the shared scope; `namespace` is always the Workspace's. ## Rules for namespace-scoped pages - Use only metrics whose `namespace` label is the tenant's namespace: kube-state-metrics, cAdvisor and kubelet volume metrics, vLLM, KubeVirt VMI metrics (`namespace`, not `exported_namespace`). -- Node, GPU (DCGM, HAMi), Ceph and KubeVirt component metrics are infrastructure: their `namespace` is the exporter's. A regular user can never see them, so hide the component when `canViewClusterMetrics` is false — do not show an empty chart or a 403. +- Node, GPU (DCGM, HAMi), Ceph and KubeVirt component metrics are infrastructure: their `namespace` is the exporter's, and their series can name other tenants' pods. On a Workspace page, take these numbers from a shared query (`SharedQueryName` lists them) for every viewer, admins included, so both see the same numbers. If no query fits, the new one goes into the server's catalog (`internal/handler/proxy_shared_catalog.go`), behind its safety test and a security review — never PromQL sent from here. Otherwise hide the component when `canViewClusterMetrics` is false: no empty chart, no 403. - Do not select on `namespace` yourself with `=~`, `!=` or `!~` against other namespaces: the server refuses a selector that cannot match the enforced namespace with 400. - Build an optional namespace filter with `namespaceSelector(ns)` (`src/lib/prometheus.ts`): it escapes the value and leaves the matcher out for an admin's empty "all namespaces" choice, where `namespace=""` would match only series without the label. - Testing with an admin account proves nothing here: an admin keeps cluster scope. Run `e2e/tenant-metrics.test.ts` as a regular Workspace member (see the comment at its top). diff --git a/src/lib/prometheus.spec.ts b/src/lib/prometheus.spec.ts index d8630389..94e3e1dd 100644 --- a/src/lib/prometheus.spec.ts +++ b/src/lib/prometheus.spec.ts @@ -9,6 +9,8 @@ import { dashboardBody, namespacePrometheusDriver, namespaceSelector, + sharedMetrics, + type SharedQueryName, workspacePrometheusDriver } from './prometheus'; @@ -67,6 +69,18 @@ describe('prometheus drivers', () => { server = createServer((req: IncomingMessage, res) => { const url = new URL(req.url ?? '/', 'http://x'); seen.push(url); + if (url.pathname.endsWith('/prometheus-shared/broken')) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + status: 'error', + errorType: 'unavailable', + error: 'cannot check metrics access on this cluster; retry shortly' + }) + ); + return; + } // series and label endpoints answer with a bare array, query endpoints with a result. const data = /\/(series|label\/)/.test(url.pathname) ? [] @@ -110,6 +124,51 @@ describe('prometheus drivers', () => { expect(workspacePrometheusDriver('c1', '', false)).toBeNull(); expect(workspacePrometheusDriver('c1', '', true)).not.toBeNull(); }); + + it('shared scope sends a name and parameters, never PromQL', async () => { + seen.length = 0; + const shared = sharedMetrics('c1', 'ws-a', origin); + const result = await shared.instant( + 'gpu-util', + { uuid: ['GPU-a', 'GPU-b'] }, + new Date(1_700_000_000_000) + ); + await shared.range( + 'host-gpu-fb-used', + { hostname: ['hci'] }, + new Date(0), + new Date(60_000), + 120 + ); + + expect(result.resultType).toBe('vector'); + expect(seen.map((u) => u.pathname)).toEqual([ + '/proxy/c1/prometheus-shared/gpu-util', + '/proxy/c1/prometheus-shared/host-gpu-fb-used' + ]); + expect(seen[0].searchParams.getAll('uuid')).toEqual(['GPU-a', 'GPU-b']); + expect(seen[0].searchParams.get('time')).toBe('1700000000'); + expect(seen[1].searchParams.getAll('hostname')).toEqual(['hci']); + expect(seen[1].searchParams.get('start')).toBe('0'); + expect(seen[1].searchParams.get('end')).toBe('60'); + expect(seen[1].searchParams.get('step')).toBe('120'); + for (const u of seen) { + expect(u.searchParams.getAll('namespace')).toEqual(['ws-a']); + expect(u.searchParams.has('query')).toBe(false); + } + }); + + it('shared scope leaves namespace out when the Workspace has none', async () => { + seen.length = 0; + await sharedMetrics('c1', '', origin).instant('kubevirt-handlers'); + expect([...seen[0].searchParams.keys()]).toEqual([]); + }); + + it("shared scope rejects with the server's message", async () => { + await expect( + sharedMetrics('c1', 'ws-a', origin).instant('broken' as SharedQueryName) + ).rejects.toThrow('cannot check metrics access'); + }); }); describe('dashboardBody', () => { diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 59f01eb8..902156bb 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -1,4 +1,4 @@ -import { InstantVector, PrometheusDriver, RangeVector } from 'prometheus-query'; +import { InstantVector, PrometheusDriver, QueryResult, RangeVector } from 'prometheus-query'; import type { ChartConfig } from '$lib/components/ui/chart/index.js'; import { m } from '$lib/messages'; @@ -15,6 +15,11 @@ import { m } from '$lib/messages'; * to namespace="" the way prom-label-proxy does it. A namespace-scoped page * therefore only works with metrics whose `namespace` label is the tenant's — * see docs/metrics.md. + * + * Shared scope (`/proxy//prometheus-shared/`) takes no PromQL: + * the server runs one of its own named queries — cluster-wide numbers that + * name no workload, per GPU card, per host, or a total — for the namespace + * scope's callers as well as cluster viewers. See sharedMetrics. */ const PROXY_HEADERS = { 'x-proxy-target': 'api' }; @@ -89,6 +94,91 @@ export function dashboardBody(state: { return state.namespace ? 'empty' : 'no-namespace'; } +/** The server's named cluster-wide queries (otterscale internal/handler/proxy_shared_catalog.go). */ +export type SharedQueryName = + | 'gpu-util' + | 'gpu-fb-used' + | 'gpu-fb-total' + | 'gpu-inventory' + | 'vgpu-memory-limit' + | 'vgpu-memory-used' + | 'host-gpu-fb-used' + | 'host-gpu-fb-total' + | 'host-gpu-count' + | 'gpu-xid-total' + | 'gpu-xid-by-gpu' + | 'kubevirt-handlers' + | 'kubevirt-controllers' + | 'kubevirt-controllers-ready' + | 'ceph-total-bytes' + | 'gpu-allocated-memory' + | 'gpu-allocated-cores' + | 'gpu-shared-count'; + +/** Values for the one repeatable parameter a shared query takes, if any. */ +export type SharedParams = { uuid?: string[]; hostname?: string[] }; + +export interface SharedMetrics { + instant(name: SharedQueryName, params?: SharedParams, time?: Date): Promise; + range( + name: SharedQueryName, + params: SharedParams, + start: Date, + end: Date, + step: number + ): Promise; +} + +/** + * Shared scope: cluster-wide numbers that name no workload — per GPU card, per + * host, or a total — for Workspace members and cluster viewers alike. The + * server holds the PromQL; this sends a name and parameters. `namespace` is the + * Workspace's and only proves membership, so an admin's "all namespaces" choice + * does not change it; it is left out when the Workspace has none, and then only + * cluster viewers get an answer. Failures reject with the server's message. + */ +export function sharedMetrics(cluster: string, namespace: string, origin = ''): SharedMetrics { + async function get(name: SharedQueryName, search: URLSearchParams): Promise { + if (namespace) search.set('namespace', namespace); + const response = await fetch(`${origin}/proxy/${cluster}/prometheus-shared/${name}?${search}`, { + headers: PROXY_HEADERS + }); + const body = (await response.json().catch(() => null)) as { + status?: string; + data?: object; + error?: string; + } | null; + if (!response.ok || body?.status !== 'success' || !body.data) { + throw new Error(`${name}: ${body?.error ?? `HTTP ${response.status}`}`); + } + return QueryResult.fromJSON(body.data); + } + + function searchFor(params: SharedParams = {}): URLSearchParams { + const search = new URLSearchParams(); + for (const uuid of params.uuid ?? []) search.append('uuid', uuid); + for (const hostname of params.hostname ?? []) search.append('hostname', hostname); + return search; + } + + const seconds = (date: Date) => String(date.getTime() / 1000); + + return { + instant(name, params, time) { + const search = searchFor(params); + if (time) search.set('time', seconds(time)); + return get(name, search); + }, + range(name, params, start, end, step) { + const search = searchFor(params); + search.set('start', seconds(start)); + search.set('end', seconds(end)); + search.set('step', String(step)); + return get(name, search); + } + }; +} + /** Escape a value for use inside PromQL double-quoted string literals (e.g. `namespace="..."`). */ export function escapePromqlStringLiteral(value: string): string { return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n'); From 88d49934e036d30fc3821ce9d0f8469160ca9510 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 14:07:19 +0800 Subject: [PATCH 24/41] feat(dashboard): show kubevirt and storage capacity cards to members - the kubevirt component tiles ask the shared scope for every viewer - so does the overview's storage card for the cluster's capacity --- .../components/dashboard/compute/index.svelte | 7 +++++-- .../compute/overview/controller.svelte | 17 ++++++--------- .../compute/overview/widgets.spec.ts | 21 ++++++++++--------- .../dashboard/compute/overview/widgets.ts | 11 +++++----- .../dashboard/compute/overview/worker.svelte | 11 +++++----- .../dashboard/workspace/index.svelte | 5 +++-- .../workspace/overview/storage.svelte | 15 +++++++------ 7 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/lib/components/dashboard/compute/index.svelte b/src/lib/components/dashboard/compute/index.svelte index e558127a..b106b765 100644 --- a/src/lib/components/dashboard/compute/index.svelte +++ b/src/lib/components/dashboard/compute/index.svelte @@ -16,7 +16,7 @@ import * as Item from '$lib/components/ui/item'; import * as Tabs from '$lib/components/ui/tabs'; import { m } from '$lib/messages'; - import { dashboardBody, workspacePrometheusDriver } from '$lib/prometheus'; + import { dashboardBody, sharedMetrics, workspacePrometheusDriver } from '$lib/prometheus'; import { Dashboard as AnalyticsDashboard } from './analytics/index'; import VmPicker from './analytics/vm-picker.svelte'; @@ -49,6 +49,9 @@ let isReloading = $state(true); let prometheusDriver = $state(null); + // KubeVirt component tiles ask the shared scope as the Workspace's member, + // whichever namespace an admin picks for the charts. + const shared = $derived(sharedMetrics(cluster, defaultNamespace)); let initialized = $state(false); const body = $derived( dashboardBody({ @@ -145,7 +148,7 @@ {#key `${selectedNamespace ?? ''}-${dashboardTimeRangeKey}`} { - it('keeps the KubeVirt component tiles for cluster viewers', () => { - expect(computeOverviewWidgets(true).map((w) => w.key)).toContain('controller'); - expect(computeOverviewWidgets(true).map((w) => w.key)).toContain('worker'); - }); +const shared = {} as SharedMetrics; - it('drops them for everyone else', () => { - const keys = computeOverviewWidgets(false).map((w) => w.key); - expect(keys).not.toContain('controller'); - expect(keys).not.toContain('worker'); - expect(keys).toContain('cpu'); +describe('computeOverviewWidgets', () => { + it('keeps the KubeVirt component tiles for everyone, fed by the shared scope', () => { + const widgets = computeOverviewWidgets(shared); + for (const key of ['controller', 'worker']) { + expect(widgets.find((w) => w.key === key)?.props).toEqual({ shared }); + } + expect(widgets.map((w) => w.key)).toContain('cpu'); + expect(widgets.find((w) => w.key === 'cpu')?.props).toBeUndefined(); }); }); diff --git a/src/lib/components/dashboard/compute/overview/widgets.ts b/src/lib/components/dashboard/compute/overview/widgets.ts index 1807354a..31a73175 100644 --- a/src/lib/components/dashboard/compute/overview/widgets.ts +++ b/src/lib/components/dashboard/compute/overview/widgets.ts @@ -1,4 +1,5 @@ import type { WidgetDefinition } from '$lib/components/custom/widget-grid'; +import type { SharedMetrics } from '$lib/prometheus'; import Controller from './controller.svelte'; import CPU from './cpu.svelte'; @@ -20,11 +21,9 @@ export const widgets = [ { key: 'throughput', class: 'col-span-6', component: Throughtput, needsNamespace: true } ] satisfies WidgetDefinition[]; -/** KubeVirt's own components: cluster-scope numbers a tenant's namespace view cannot show. */ -const clusterScopeWidgetKeys = new Set(['controller', 'worker']); +/** KubeVirt's own components: cluster-wide numbers the server answers as shared queries. */ +const sharedWidgetKeys = new Set(['controller', 'worker']); -export function computeOverviewWidgets(canViewClusterMetrics: boolean): WidgetDefinition[] { - return canViewClusterMetrics - ? widgets - : widgets.filter((w) => !clusterScopeWidgetKeys.has(w.key)); +export function computeOverviewWidgets(shared: SharedMetrics): WidgetDefinition[] { + return widgets.map((w) => (sharedWidgetKeys.has(w.key) ? { ...w, props: { shared } } : w)); } diff --git a/src/lib/components/dashboard/compute/overview/worker.svelte b/src/lib/components/dashboard/compute/overview/worker.svelte index 7805b8ba..768b0a6c 100644 --- a/src/lib/components/dashboard/compute/overview/worker.svelte +++ b/src/lib/components/dashboard/compute/overview/worker.svelte @@ -2,21 +2,20 @@ import BoxIcon from '@lucide/svelte/icons/box'; import ChartColumnIcon from '@lucide/svelte/icons/chart-column'; import Loader2Icon from '@lucide/svelte/icons/loader-2'; - import type { PrometheusDriver, SampleValue } from 'prometheus-query'; + import type { SampleValue } from 'prometheus-query'; import { onDestroy, onMount } from 'svelte'; import { ReloadManager } from '$lib/components/custom/reloader'; import * as Card from '$lib/components/ui/card'; import { m } from '$lib/messages'; + import type { SharedMetrics } from '$lib/prometheus'; - let { - prometheusDriver, - isReloading = $bindable() - }: { prometheusDriver: PrometheusDriver; isReloading: boolean } = $props(); + let { shared, isReloading = $bindable() }: { shared: SharedMetrics; isReloading: boolean } = + $props(); let workers: SampleValue | undefined = $state(undefined); async function fetchWorkers() { - const response = await prometheusDriver.instantQuery(`count(kubevirt_info{})`); + const response = await shared.instant('kubevirt-handlers'); workers = response.result[0]?.value ?? undefined; } diff --git a/src/lib/components/dashboard/workspace/index.svelte b/src/lib/components/dashboard/workspace/index.svelte index a0266a70..d66b086f 100644 --- a/src/lib/components/dashboard/workspace/index.svelte +++ b/src/lib/components/dashboard/workspace/index.svelte @@ -14,7 +14,7 @@ import * as Item from '$lib/components/ui/item'; import * as Tabs from '$lib/components/ui/tabs'; import { m } from '$lib/messages'; - import { dashboardBody, workspacePrometheusDriver } from '$lib/prometheus'; + import { dashboardBody, sharedMetrics, workspacePrometheusDriver } from '$lib/prometheus'; import { workspaceOverviewWidgets } from './overview/widgets'; @@ -44,10 +44,11 @@ let pickerTo = $state(nowCDT()); let pickerToIsNow = $state(true); + const shared = $derived(sharedMetrics(cluster, namespace)); const widgets = $derived( workspaceOverviewWidgets.map((w) => ({ ...w, - props: w.key === 'storage' ? { namespace, canViewClusterMetrics } : { namespace } + props: w.key === 'storage' ? { namespace, shared } : { namespace } })) ); diff --git a/src/lib/components/dashboard/workspace/overview/storage.svelte b/src/lib/components/dashboard/workspace/overview/storage.svelte index bafa88f5..c1a15637 100644 --- a/src/lib/components/dashboard/workspace/overview/storage.svelte +++ b/src/lib/components/dashboard/workspace/overview/storage.svelte @@ -13,18 +13,18 @@ import { Progress } from '$lib/components/ui/progress'; import { formatCapacity } from '$lib/formatter'; import { m } from '$lib/messages'; - import { escapePromqlStringLiteral } from '$lib/prometheus'; + import { escapePromqlStringLiteral, type SharedMetrics } from '$lib/prometheus'; import { cn } from '$lib/utils'; let { prometheusDriver, namespace, - canViewClusterMetrics = false, + shared, isReloading = $bindable() }: { prometheusDriver: PrometheusDriver; namespace: string; - canViewClusterMetrics?: boolean; + shared: SharedMetrics; isReloading: boolean; } = $props(); @@ -53,9 +53,12 @@ `sum(kubelet_volume_stats_used_bytes{namespace="${ns}"}) or vector(0)`, evalAt ), - canViewClusterMetrics - ? prometheusDriver.instantQuery(`sum(ceph_cluster_total_bytes) or vector(0)`, evalAt) - : Promise.resolve(undefined), + // The cluster's capacity, to say when requests exceed it. Best effort: + // without it the card still shows the Workspace's own numbers. + shared.instant('ceph-total-bytes', undefined, evalAt).catch((error) => { + console.warn('Failed to fetch cluster storage capacity:', error); + return undefined; + }), prometheusDriver.instantQuery( `count(kube_persistentvolumeclaim_info{namespace="${ns}"}) or vector(0)`, evalAt From 372ac47e6c8af1ea0fba0af07ae07eae0e11af96 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 14:11:13 +0800 Subject: [PATCH 25/41] feat(dashboard): show gpu resources in the update dialog to members - per-card inventory and vgpu totals come from the shared scope --- .../utils/gpu-resources.svelte | 370 +++++++++--------- 1 file changed, 183 insertions(+), 187 deletions(-) diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte index 241a0065..2d064b0f 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte @@ -10,24 +10,22 @@ import * as Modal from '$lib/components/ui/dialog'; import * as Item from '$lib/components/ui/item'; import * as Table from '$lib/components/ui/table'; - import { clusterPrometheusDriver } from '$lib/prometheus'; + import { sharedMetrics } from '$lib/prometheus'; let gpus: Record[] = []; let gpusByModelName: Record[] = []; let gpusByHostName: Record[] = []; onMount(async () => { - // DCGM and HAMi are cluster scope: a regular user would only get 403s. - if (page.data.canViewClusterMetrics !== true) return; - const driver = clusterPrometheusDriver(page.params.cluster!); + // Per-card inventory and vGPU totals: cluster-wide, naming no workload, + // from the shared scope for every viewer. + const shared = sharedMetrics(page.params.cluster!, page.data.namespace ?? ''); try { const [fieldBufferResponse, vgpuLimitResponse, vgpuUsageResponse] = await Promise.all([ - driver.instantQuery( - '(DCGM_FI_DEV_FB_FREE + DCGM_FI_DEV_FB_RESERVED + DCGM_FI_DEV_FB_USED) * (1024 * 1024)' // DCGM return in Mi - ), - driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_limit_bytes)'), - driver.instantQuery('sum by (device_uuid) (hami_vgpu_memory_used_bytes)') + shared.instant('gpu-inventory'), + shared.instant('vgpu-memory-limit'), + shared.instant('vgpu-memory-used') ]); const vgpuLimits = vgpuLimitResponse.result.map((series) => { @@ -79,187 +77,185 @@ }); -{#if page.data.canViewClusterMetrics === true} - - - - - e.preventDefault()} - > - - - GPU Resources - Amount of limits, usage, and total field buffers. - - {gpus.length} devices - + + + + + e.preventDefault()} + > + + + GPU Resources + Amount of limits, usage, and total field buffers. + + {gpus.length} devices + - - - Summary - - {gpusByModelName.length} types - - - - - - - Identifier - usage - limit - total + + + Summary + + {gpusByModelName.length} types + + + + + + + Identifier + usage + limit + total + + + + {#each gpusByModelName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + - - - {#each gpusByModelName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {/each} - - + {/each} + + - - - Summary - - {gpusByHostName.length} hosts - - - - - - - Identifier - usage - limit - total + + + Summary + + {gpusByHostName.length} hosts + + + + + + + Identifier + usage + limit + total + + + + {#each gpusByHostName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + - - - {#each gpusByHostName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {/each} - - + {/each} + + - - - Details - - {gpus.length} devices - - - - - - - Model Name - Host Name - Device - UUID - usage - limit - total + + + Details + + {gpus.length} devices + + + + + + + Model Name + Host Name + Device + UUID + usage + limit + total + + + + {#each gpus as gpu, index (index)} + + {gpu.modelName} + {gpu.hostname} + {gpu.device} + {gpu.UUID} + + {@const { value, unit } = + gpu.usage && typeof gpu.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpu.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.limit && typeof gpu.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpu.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.total && typeof gpu.total === 'number' + ? formatWithBinarySuffix(BigInt(gpu.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + - - - {#each gpus as gpu, index (index)} - - {gpu.modelName} - {gpu.hostname} - {gpu.device} - {gpu.UUID} - - {@const { value, unit } = - gpu.usage && typeof gpu.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpu.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.limit && typeof gpu.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpu.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.total && typeof gpu.total === 'number' - ? formatWithBinarySuffix(BigInt(gpu.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {/each} - - - - -{/if} + {/each} + + + + From d6ef5f74b0693a09851361603b33491d235d13b7 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 14:24:09 +0800 Subject: [PATCH 26/41] feat(dashboard): show a model's gpu memory and xid to members - the model's nodes come from its namespace, as before - their gpu numbers come from the shared scope, for every viewer - xid shows as unknown, not 0, when it cannot be asked - drop vllmModelHostnamesSelector, which nothing uses any more --- messages/en.json | 1 + messages/zh-hant.json | 1 + .../model/analytics/b-gpu-memory.svelte | 34 ++-- .../model/analytics/b-pod-health.svelte | 151 +++++++++--------- .../dashboard/model/analytics/index.svelte | 30 ++-- .../components/dashboard/model/index.svelte | 7 +- src/lib/prometheus.ts | 28 +--- 7 files changed, 125 insertions(+), 127 deletions(-) diff --git a/messages/en.json b/messages/en.json index 1f53c73a..063cbbe4 100644 --- a/messages/en.json +++ b/messages/en.json @@ -641,6 +641,7 @@ "metric_restart": "Restart", "metric_oom": "OOM", "metric_xid": "XID", + "xid_unavailable": "Couldn't load XID errors right now. Refresh to try again.", "metric_cpu_throttle": "Throttle", "metric_shm_crash": "Shm", "metric_preempt": "Preempt", diff --git a/messages/zh-hant.json b/messages/zh-hant.json index 5b72931a..6f8f1d49 100644 --- a/messages/zh-hant.json +++ b/messages/zh-hant.json @@ -641,6 +641,7 @@ "metric_restart": "重啟", "metric_oom": "OOM", "metric_xid": "XID", + "xid_unavailable": "暫時無法取得 XID 錯誤，請重新整理再試一次。", "metric_cpu_throttle": "節流", "metric_shm_crash": "Shm", "metric_preempt": "搶占", diff --git a/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte b/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte index 38427988..719da722 100644 --- a/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte +++ b/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte @@ -14,10 +14,16 @@ import * as Chart from '$lib/components/ui/chart'; import * as Tooltip from '$lib/components/ui/tooltip'; import { m } from '$lib/messages'; - import { computeStep, type DataPoint, vllmModelHostnamesSelector } from '$lib/prometheus'; + import { + computeStep, + type DataPoint, + fetchModelNodes, + type SharedMetrics + } from '$lib/prometheus'; let { prometheusDriver, + shared, namespace, selectedModel, start, @@ -26,6 +32,7 @@ isReloading = $bindable() }: { prometheusDriver: PrometheusDriver; + shared: SharedMetrics; namespace: string | undefined; selectedModel: string; start: Date; @@ -67,17 +74,22 @@ const endMs = endIsNow ? Date.now() : end.getTime(); const step = computeStep(startMs, endMs); - // Inline the node filter so Prometheus does the model→host join itself — - // avoids a serial `fetchModelNodes` round-trip before these queries fire. - const hostFilter = vllmModelHostnamesSelector(namespace, selectedModel); - const usedQuery = `sum by(hostname) (DCGM_FI_DEV_FB_USED and on(hostname) (${hostFilter}))`; - const totalQuery = `sum by(hostname) ((DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) and on(hostname) (${hostFilter}))`; - const countQuery = `count by(hostname) (DCGM_FI_DEV_FB_USED and on(hostname) (${hostFilter}))`; - + // The model's nodes come from its own namespace; their GPUs' memory is + // cluster-wide and comes from the shared scope, per host. + const nodes = await fetchModelNodes(prometheusDriver, namespace, selectedModel); + if (nodes.length === 0) { + nodeCount = 0; + data = []; + hosts = []; + hostInfo = {}; + lookup = {}; + return; + } + const params = { hostname: nodes }; const [usedResp, totalResp, countResp] = await Promise.all([ - prometheusDriver.rangeQuery(usedQuery, new Date(startMs), new Date(endMs), `${step}s`), - prometheusDriver.instantQuery(totalQuery), - prometheusDriver.instantQuery(countQuery) + shared.range('host-gpu-fb-used', params, new Date(startMs), new Date(endMs), step), + shared.instant('host-gpu-fb-total', params), + shared.instant('host-gpu-count', params) ]); const gpuCountByHost: Record = {}; diff --git a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte index 67b29bd6..fee38d47 100644 --- a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte +++ b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte @@ -1,7 +1,7 @@ -{#if canViewClusterMetrics} +{#if available} {#if trigger} diff --git a/src/lib/components/gpu-allocation/types.ts b/src/lib/components/gpu-allocation/types.ts index 54711627..6efd06be 100644 --- a/src/lib/components/gpu-allocation/types.ts +++ b/src/lib/components/gpu-allocation/types.ts @@ -58,12 +58,24 @@ export interface GpuUtilization { totalMem?: number; } +/** + * What hami-scheduler has booked on a card across every workload: memory in MiB + * (the unit GpuDevice speaks) and how many containers share it. Names no pod — + * it is what a viewer who may not list other tenants' pods sees instead. + */ +export interface GpuAllocatedTotal { + usedMem: number; + shareCount: number; +} + export interface GpuInfo { device: GpuDevice; nodeName: string; allocatedBy: { podName: string; podNamespace: string; usedCores: number; usedMem: number }[]; /** Measured usage of the card; absent when DCGM reports nothing for it. */ utilization?: GpuUtilization; + /** Per-card totals for viewers who may not list every pod on the node. */ + allocatedTotal?: GpuAllocatedTotal; } export interface NodeInfo { diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte index e1dad7b7..df5768d9 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/llm-inference-service/actions.svelte @@ -4,7 +4,6 @@ import type { Schema } from '@sjsf/form'; import type { ValidateFunction } from 'ajv'; - import { page } from '$app/state'; import Delete from '$lib/components/kind-viewer/kind-viewer-actions/default/delete.svelte'; import Describe from '$lib/components/kind-viewer/kind-viewer-actions/default/describe.svelte'; import Edit from '$lib/components/kind-viewer/kind-viewer-actions/default/edit.svelte'; @@ -53,23 +52,20 @@ Inspect - - {#if page.data.canViewClusterMetrics === true} - { - e.preventDefault(); + { + e.preventDefault(); + }} + > + { + actionsOpen = false; }} - > - { - actionsOpen = false; - }} - /> - - {/if} + /> + { e.preventDefault(); From 5f9bf72671cce6972ab4114646a0eab402532d03 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 14:33:16 +0800 Subject: [PATCH 28/41] test(dashboard): check members get shared numbers and nothing more - gpu, kubevirt and ceph components may only use the shared scope - e2e follows every shared call: known name, answered, own namespace - e2e opens the model's analytics, gpu allocation and gpu resources --- e2e/tenant-metrics.test.ts | 133 ++++++++++++++++-- .../dashboard/namespace-scope.spec.ts | 27 ++++ 2 files changed, 151 insertions(+), 9 deletions(-) diff --git a/e2e/tenant-metrics.test.ts b/e2e/tenant-metrics.test.ts index 9d4af21d..723c15d1 100644 --- a/e2e/tenant-metrics.test.ts +++ b/e2e/tenant-metrics.test.ts @@ -5,13 +5,16 @@ * E2E_BASE_URL=https:// E2E_IGNORE_HTTPS_ERRORS=1 \ * E2E_TENANT_USERNAME=... E2E_TENANT_PASSWORD=... \ * E2E_CLUSTER= E2E_WORKSPACE= E2E_NAMESPACE= \ + * [E2E_MODEL=] \ * pnpm exec playwright test e2e/tenant-metrics.test.ts * * The user must be a plain Workspace member (view is enough): not cluster-admin, * not bound to otterscale:metrics-cluster-view, in no Keycloak group that * grants either. An admin account proves nothing here — admins keep cluster * scope. E2E_EXPECT_VM_DATA / E2E_EXPECT_MODEL_DATA=1 also require data on the - * compute / model pages (set them when the namespace runs a VM / a model). + * compute / model pages (set them when the namespace runs a VM / a model); + * E2E_MODEL names the model for the analytics and model-service tests, which + * are skipped without it. */ import { expect, type Page, test } from '@playwright/test'; @@ -23,6 +26,31 @@ const env = { namespace: process.env.E2E_NAMESPACE ?? '' }; +/** Optional: the model whose analytics and actions are checked. */ +const model = process.env.E2E_MODEL ?? ''; + +/** The server's named queries (otterscale internal/handler/proxy_shared_catalog.go). */ +const SHARED_NAMES = new Set([ + 'gpu-util', + 'gpu-fb-used', + 'gpu-fb-total', + 'gpu-inventory', + 'vgpu-memory-limit', + 'vgpu-memory-used', + 'host-gpu-fb-used', + 'host-gpu-fb-total', + 'host-gpu-count', + 'gpu-xid-total', + 'gpu-xid-by-gpu', + 'kubevirt-handlers', + 'kubevirt-controllers', + 'kubevirt-controllers-ready', + 'ceph-total-bytes', + 'gpu-allocated-memory', + 'gpu-allocated-cores', + 'gpu-shared-count' +]); + test.skip( !process.env.E2E_BASE_URL || Object.values(env).some((v) => !v), 'needs E2E_BASE_URL and the E2E_TENANT_* / E2E_CLUSTER / E2E_WORKSPACE / E2E_NAMESPACE variables' @@ -37,8 +65,9 @@ type Sample = { values?: [number, string][]; }; type Captured = { url: URL; status: number; result: Sample[] }; +type SharedCall = { name: string; status: number; params: URLSearchParams }; -/** Infrastructure metrics a namespace-scoped page must not even ask for. */ +/** Infrastructure metrics: never in a namespace-scoped query; the shared scope serves them by name. */ const INFRASTRUCTURE = [ 'ceph_cluster_total_bytes', 'kubevirt_info', @@ -49,12 +78,22 @@ const INFRASTRUCTURE = [ function watchMetrics(page: Page) { const tenancy: Captured[] = []; + const shared: SharedCall[] = []; const clusterScope: string[] = []; + const sharedPrefix = `/proxy/${env.cluster}/prometheus-shared/`; page.on('request', (req) => { if (req.url().includes(`/proxy/${env.cluster}/prometheus/`)) clusterScope.push(req.url()); }); page.on('response', async (res) => { const url = new URL(res.url()); + if (url.pathname.startsWith(sharedPrefix)) { + shared.push({ + name: url.pathname.slice(sharedPrefix.length), + status: res.status(), + params: url.searchParams + }); + return; + } if (!url.pathname.startsWith(`/proxy/${env.cluster}/prometheus-tenancy/`)) return; let result: Sample[] = []; try { @@ -65,7 +104,18 @@ function watchMetrics(page: Page) { } tenancy.push({ url, status: res.status(), result }); }); - return { tenancy, clusterScope }; + return { tenancy, shared, clusterScope }; +} + +/** Every shared call is a known name, answered, asked as this Workspace; `names` were all asked. */ +function expectShared(calls: SharedCall[], names: string[]) { + for (const c of calls) { + expect(SHARED_NAMES.has(c.name), `unknown shared query ${c.name}`).toBe(true); + expect(c.status, `${c.name} answered ${c.status}`).toBeLessThan(400); + expect(c.params.getAll('namespace')).toEqual([env.namespace]); + } + const asked = new Set(calls.map((c) => c.name)); + for (const name of names) expect(asked.has(name), `${name} was not asked`).toBe(true); } const sentQueries = (captured: Captured[]) => @@ -147,31 +197,96 @@ for (const name of ['cluster', 'storage']) { }); } -test('overview shows their namespace, with data, and nothing cluster-wide', async ({ page }) => { - const { tenancy, clusterScope } = await visit(page, 'overview'); +test('overview shows their namespace, and the cluster capacity through the shared scope', async ({ + page +}) => { + const { tenancy, shared, clusterScope } = await visit(page, 'overview'); expectOwnNamespaceOnly(tenancy); expect(hasData(tenancy), 'overview charts are empty').toBe(true); expect(clusterScope).toEqual([]); for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); + expectShared(shared, ['ceph-total-bytes']); }); -test('compute stays in their namespace and hides KubeVirt components', async ({ page }) => { - const { tenancy, clusterScope } = await visit(page, 'compute'); +test('compute stays in their namespace and shows KubeVirt components', async ({ page }) => { + const { tenancy, shared, clusterScope } = await visit(page, 'compute'); expectOwnNamespaceOnly(tenancy); expect(clusterScope).toEqual([]); for (const q of sentQueries(tenancy)) { for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); expect(q).not.toContain('exported_namespace'); } + expectShared(shared, ['kubevirt-handlers', 'kubevirt-controllers', 'kubevirt-controllers-ready']); if (process.env.E2E_EXPECT_VM_DATA === '1') expect(hasData(tenancy), 'compute charts are empty').toBe(true); }); -test('model stays in their namespace and hides GPU memory and XID', async ({ page }) => { - const { tenancy, clusterScope } = await visit(page, 'model'); +test('model overview stays in their namespace', async ({ page }) => { + const { tenancy, shared, clusterScope } = await visit(page, 'model'); expectOwnNamespaceOnly(tenancy); expect(clusterScope).toEqual([]); for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); + expectShared(shared, []); if (process.env.E2E_EXPECT_MODEL_DATA === '1') expect(hasData(tenancy), 'model charts are empty').toBe(true); }); + +test('model analytics shows GPU memory and XID through the shared scope', async ({ page }) => { + test.skip(!model, 'needs E2E_MODEL'); + const watched = watchMetrics(page); + await page.goto(dashboard('model')); + await page.getByRole('tab', { name: /^(Analytics|分析)$/ }).click(); + await page.getByRole('combobox').first().click(); + await page.getByRole('option', { name: model, exact: true }).click(); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(3_000); // panels query after the model is picked + + await expect( + page.getByText(/GPU Memory: Allocated vs Used|GPU 記憶體：已分配 vs 已使用/) + ).toBeVisible(); + await expect(page.getByText('XID', { exact: true })).toBeVisible(); + expectOwnNamespaceOnly(watched.tenancy); + expect(watched.clusterScope).toEqual([]); + expectShared(watched.shared, [ + 'host-gpu-fb-used', + 'host-gpu-fb-total', + 'host-gpu-count', + 'gpu-xid-total', + 'gpu-xid-by-gpu' + ]); +}); + +test('a model service shows its GPU allocation and the GPU resources', async ({ page }) => { + test.skip(!model, 'needs E2E_MODEL'); + const watched = watchMetrics(page); + const list = `/${env.cluster}/${env.workspace}?group=serving.kserve.io&version=v1alpha2&kind=LLMInferenceService&resource=llminferenceservices`; + const openActions = async () => { + await page + .getByRole('row', { name: new RegExp(model) }) + .getByRole('button') + .last() + .click(); + }; + + await page.goto(list); + await openActions(); + await page.getByRole('menuitem', { name: /^(GPU Allocation|GPU 分配)$/ }).click(); + await expect(page.getByRole('dialog')).toBeVisible(); + await page.waitForLoadState('networkidle'); + expectShared(watched.shared, [ + 'gpu-util', + 'gpu-fb-used', + 'gpu-fb-total', + 'gpu-allocated-memory', + 'gpu-shared-count' + ]); + await page.keyboard.press('Escape'); + + await openActions(); + await page.getByRole('menuitem', { name: /^Update$/ }).click(); + await page.locator('button:has(svg.lucide-gpu)').first().click(); + await expect(page.getByText('GPU Resources', { exact: true })).toBeVisible(); + await page.waitForLoadState('networkidle'); + expectShared(watched.shared, ['gpu-inventory', 'vgpu-memory-limit', 'vgpu-memory-used']); + expect(watched.clusterScope).toEqual([]); +}); diff --git a/src/lib/components/dashboard/namespace-scope.spec.ts b/src/lib/components/dashboard/namespace-scope.spec.ts index e06f4d32..9dc3e41b 100644 --- a/src/lib/components/dashboard/namespace-scope.spec.ts +++ b/src/lib/components/dashboard/namespace-scope.spec.ts @@ -28,3 +28,30 @@ describe('compute dashboard', () => { expect(offenders).toEqual([]); }); }); + +// GPU, KubeVirt and Ceph numbers on Workspace pages come from the server's +// named queries (shared scope), for admins and members alike: these files pick +// no cluster-scope driver and carry no infrastructure PromQL of their own. +describe('shared-scope components', () => { + const sharedFiles = [ + 'src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte', + 'src/lib/components/gpu-allocation/fetch-utilization.ts', + 'src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte', + 'src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte', + 'src/lib/components/dashboard/model/analytics/b-pod-health.svelte', + 'src/lib/components/dashboard/compute/overview/controller.svelte', + 'src/lib/components/dashboard/compute/overview/worker.svelte', + 'src/lib/components/dashboard/workspace/overview/storage.svelte' + ]; + const infrastructure = + /DCGM_FI_|hami_|kubevirt_info|kubevirt_virt_controller|ceph_cluster_total_bytes/; + + for (const file of sharedFiles) { + it(`${file} uses the shared scope`, () => { + const source = readFileSync(join(process.cwd(), file), 'utf8'); + expect(source).not.toContain('clusterPrometheusDriver'); + expect(source).not.toMatch(infrastructure); + expect(source).toMatch(/\bshared\b/); + }); + } +}); From 9d95df5036b8f2df145208fa9847e414d58c11e3 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 14:59:02 +0800 Subject: [PATCH 29/41] fix(dashboard): tell unknown gpu numbers apart from a model without gpu - fetchModelNodes resolves null when the lookup fails, [] for no nodes - gpu memory shows no data, not no gpu, when a lookup or call fails - xid shows the grey unavailable dash when the node lookup fails - test fetchModelNodes for a failed lookup and duplicate nodes --- .../model/analytics/b-gpu-memory.svelte | 19 +++++++++++++++ .../model/analytics/b-pod-health.svelte | 7 ++++-- src/lib/prometheus.spec.ts | 24 +++++++++++++++++++ src/lib/prometheus.ts | 7 +++--- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte b/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte index 719da722..8614cc74 100644 --- a/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte +++ b/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte @@ -61,6 +61,9 @@ let hostInfo = $state>({}); let lookup = $state>>({}); let nodeCount = $state(0); + // The model's nodes or their GPUs could not be read: show "no data", not "no GPU". + // Reset only when a fetch succeeds, so a reload after a failure does not flash "no GPU". + let failed = $state(false); let isLoaded = $state(false); // Captured during labelFormatter so the value formatter can find the matching sample. @@ -77,7 +80,16 @@ // The model's nodes come from its own namespace; their GPUs' memory is // cluster-wide and comes from the shared scope, per host. const nodes = await fetchModelNodes(prometheusDriver, namespace, selectedModel); + if (nodes === null) { + failed = true; + data = []; + hosts = []; + hostInfo = {}; + lookup = {}; + return; + } if (nodes.length === 0) { + failed = false; nodeCount = 0; data = []; hosts = []; @@ -91,6 +103,7 @@ shared.instant('host-gpu-fb-total', params), shared.instant('host-gpu-count', params) ]); + failed = false; const gpuCountByHost: Record = {}; for (const v of countResp.result) { @@ -154,6 +167,7 @@ (a, b) => (a.date as Date).getTime() - (b.date as Date).getTime() ); } catch { + failed = true; data = []; hosts = []; hostInfo = {}; @@ -229,6 +243,11 @@
+ {:else if failed} +
+ +

{m.no_data_display()}

+
{:else if nodeCount === 0}
diff --git a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte index fee38d47..db265f83 100644 --- a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte +++ b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte @@ -115,7 +115,10 @@ * XID errors on the GPUs of the nodes the model runs on, from the shared scope; * null when they could not be asked, which the tile shows as unknown, not as 0. */ - async function fetchXid(nodes: string[]): Promise<{ total: number; byGpu: GpuEntry[] } | null> { + async function fetchXid( + nodes: string[] | null + ): Promise<{ total: number; byGpu: GpuEntry[] } | null> { + if (nodes === null) return null; if (nodes.length === 0) return { total: 0, byGpu: [] }; const params = { hostname: nodes }; try { @@ -185,7 +188,7 @@ throttlePct = Number.isFinite(throttleV) ? throttleV : 0; preemptTotal = Math.round(scalarFromVectors(main.preemptTotal)); preemptByPod = perPodFromVectors(main.preemptPerPod); - nodeCount = nodes.length; + nodeCount = nodes?.length ?? 0; xidUnavailable = xid === null; xidTotal = xid?.total ?? 0; xidByGpu = xid?.byGpu ?? []; diff --git a/src/lib/prometheus.spec.ts b/src/lib/prometheus.spec.ts index 94e3e1dd..ed657f82 100644 --- a/src/lib/prometheus.spec.ts +++ b/src/lib/prometheus.spec.ts @@ -1,12 +1,14 @@ import { createServer, type IncomingMessage, type Server } from 'node:http'; import type { AddressInfo } from 'node:net'; +import type { PrometheusDriver } from 'prometheus-query'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { classifyGpuGovernance, clusterPrometheusDriver, dashboardBody, + fetchModelNodes, namespacePrometheusDriver, namespaceSelector, sharedMetrics, @@ -203,3 +205,25 @@ describe('namespaceSelector', () => { expect(namespaceSelector('')).toBe(''); }); }); + +describe('fetchModelNodes', () => { + const driver = (instantQuery: () => Promise) => + ({ instantQuery }) as unknown as PrometheusDriver; + + it('resolves null when the lookup fails, not "no nodes"', async () => { + const failing = driver(async () => { + throw new Error('cannot check metrics access on this cluster; retry shortly'); + }); + await expect(fetchModelNodes(failing, 'ws-a', 'm1')).resolves.toBeNull(); + }); + + it('returns each node once', async () => { + const series = (node: string) => ({ metric: { labels: { node } } }); + const answering = driver(async () => ({ + resultType: 'vector', + result: [series('n1'), series('n1'), series('n2')] + })); + const nodes = await fetchModelNodes(answering, 'ws-a', 'm1'); + expect(nodes?.slice().sort()).toEqual(['n1', 'n2']); + }); +}); diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 13cd452b..bcf26e1d 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -731,13 +731,14 @@ export function classifyGpuGovernance(allocated?: number, used?: number): GpuGov * Joins `kube_pod_info` against `vllm:kv_cache_usage_perc` so we only keep pods * that belong to `selectedModel`, then returns the distinct `node` label values. * Used to scope DCGM/GPU queries (which lack pod-level labels) to the nodes the - * model actually occupies. + * model actually occupies. Resolves null when the lookup failed, so callers can + * tell 'unknown' from 'no GPU'. */ export async function fetchModelNodes( client: PrometheusDriver, namespace: string | undefined, selectedModel: string -): Promise { +): Promise { const ns = (namespace ?? '').trim(); const nsSel = ns ? `namespace="${escapePromqlStringLiteral(ns)}"` : ''; const podInfoSelector = nsSel ? `{${nsSel}}` : ''; @@ -756,7 +757,7 @@ export async function fetchModelNodes( } return Array.from(nodes); } catch { - return []; + return null; } } From b4a4778896aaae46c1edca659109dab9591637a6 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 14:59:02 +0800 Subject: [PATCH 30/41] docs(dashboard): say the server answers at three scopes - metrics.md and the prometheus.ts header still named two scopes --- docs/metrics.md | 2 +- src/lib/prometheus.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 84299c5c..786a22f1 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,6 +1,6 @@ # Metrics in the dashboard -The OtterScale server answers PromQL at two scopes: +The OtterScale server answers metrics at three scopes; the shared one takes no PromQL: | Scope | Path | Who | What | | --------- | ---------------------------------------------------- | ------------------------------------------------------------ | -------------------------------------------------------------------------------- | diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index bcf26e1d..d7c827c3 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -4,7 +4,7 @@ import type { ChartConfig } from '$lib/components/ui/chart/index.js'; import { m } from '$lib/messages'; /* - * The OtterScale server answers PromQL at two scopes. + * The OtterScale server answers metrics at three scopes. * * Cluster scope (`/proxy//prometheus`) takes any query, but only from * viewers with cluster metrics access (cluster-admin, or a binding to From c08c68855276ba40b7f651e96284375dfc7e6083 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 16:51:56 +0800 Subject: [PATCH 31/41] test(dashboard): look for the gpu button inside the update dialog - the row behind the dialog has gpu icons too - the dialog's overlay took the click meant for the row's icon - found when the e2e first ran on the hci lab with E2E_MODEL set --- e2e/tenant-metrics.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/tenant-metrics.test.ts b/e2e/tenant-metrics.test.ts index 723c15d1..4b3bbf97 100644 --- a/e2e/tenant-metrics.test.ts +++ b/e2e/tenant-metrics.test.ts @@ -284,7 +284,8 @@ test('a model service shows its GPU allocation and the GPU resources', async ({ await openActions(); await page.getByRole('menuitem', { name: /^Update$/ }).click(); - await page.locator('button:has(svg.lucide-gpu)').first().click(); + // The row behind the dialog has GPU icons too; the overlay would take the click. + await page.getByRole('dialog').last().locator('button:has(svg.lucide-gpu)').first().click(); await expect(page.getByText('GPU Resources', { exact: true })).toBeVisible(); await page.waitForLoadState('networkidle'); expectShared(watched.shared, ['gpu-inventory', 'vgpu-memory-limit', 'vgpu-memory-used']); From 65d8214e1592ea2feb8c81eada9a25e9f21786ec Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 19:46:39 +0800 Subject: [PATCH 32/41] fix(dashboard): use shared gpu totals unless cluster admin - list node pods only for a cluster admin, who may list every namespace - fetch gpu-allocated totals for everyone else, metrics viewers included - align card-usage and fetch-topology comments with the new check --- .../components/gpu-allocation/card-usage.ts | 8 +++---- .../gpu-allocation/fetch-topology.ts | 4 ++-- .../gpu-allocation-dialog.svelte | 23 ++++++++----------- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/lib/components/gpu-allocation/card-usage.ts b/src/lib/components/gpu-allocation/card-usage.ts index 97e05477..4910253f 100644 --- a/src/lib/components/gpu-allocation/card-usage.ts +++ b/src/lib/components/gpu-allocation/card-usage.ts @@ -1,10 +1,10 @@ import type { GpuInfo } from './types'; /** - * How full a card is, as its bar and its node's total show it. A viewer who - * may list every pod on the node sees the sum of what those pods booked; one - * who may not gets the scheduler's per-card totals, which count every tenant's - * pods without naming them. + * How full a card is, as its bar and its node's total show it. A cluster admin + * lists every pod on the node and sees the sum of what those pods booked; + * everyone else gets the scheduler's per-card totals, which count every + * tenant's pods without naming them. */ export function cardUsage(gpu: GpuInfo): { usedMem: number; shareCount: number } { if (gpu.allocatedTotal) return { ...gpu.allocatedTotal }; diff --git a/src/lib/components/gpu-allocation/fetch-topology.ts b/src/lib/components/gpu-allocation/fetch-topology.ts index 6aab559d..90ff5d2a 100644 --- a/src/lib/components/gpu-allocation/fetch-topology.ts +++ b/src/lib/components/gpu-allocation/fetch-topology.ts @@ -153,8 +153,8 @@ async function attachWorkspaces( /** * `listNodePods` lists every GPU pod on the service's nodes, so a card's usage - * counts every workload on it. Viewers without cluster metrics access skip it — - * every such request would be refused — and get per-card totals from the shared + * counts every workload on it. Only a cluster admin may list pods in every + * namespace; everyone else skips it and gets per-card totals from the shared * scope instead (gpu-allocation-dialog.svelte). */ export async function fetchLLMInferenceServiceTopology( diff --git a/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte b/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte index a9f9a036..744b53c5 100644 --- a/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte +++ b/src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte @@ -39,11 +39,14 @@ const name: string = $derived(object?.metadata?.name ?? ''); /** - * Cluster viewers see every pod on the involved nodes. Everyone else may open a - * model's view: its own pods, with per-card totals from the shared scope that - * count other tenants' bookings without naming them. The node view lists every - * pod on a node, so it stays with cluster viewers. + * A model's view is open to everyone. Only a cluster admin, who may list pods + * in every namespace, counts each card from the pods on its node; everyone else + * gets per-card totals from the shared scope, which count other tenants' + * bookings without naming them. A cluster metrics viewer may still be refused + * the pod listing, so that permission alone is not enough. The node view lists + * every pod on a node, so it stays with cluster metrics viewers. */ + const isClusterAdmin = $derived(page.data.isClusterAdmin === true); const canViewClusterMetrics = $derived(page.data.canViewClusterMetrics === true); const available = $derived(view === 'llm-inference-service' || canViewClusterMetrics); const shared = $derived(sharedMetrics(cluster, namespace)); @@ -57,7 +60,7 @@ let requestId = 0; /** - * Attach measured per-card usage, and for viewers who could not list every pod + * Attach measured per-card usage, and for viewers other than a cluster admin * the scheduler's per-card totals, to the topology already on screen. Runs after * the diagram renders rather than as part of `fetchData`, so a failing query * only costs these numbers — never the allocation view itself. @@ -71,7 +74,7 @@ shared, data.gpus.map((gpu) => gpu.device.id) ), - canViewClusterMetrics + isClusterAdmin ? Promise.resolve(new Map()) : fetchGpuAllocatedTotals(shared) ]); @@ -100,13 +103,7 @@ try { const data = view === 'llm-inference-service' - ? await fetchLLMInferenceServiceTopology( - client, - cluster, - namespace, - name, - canViewClusterMetrics - ) + ? await fetchLLMInferenceServiceTopology(client, cluster, namespace, name, isClusterAdmin) : await fetchNodeTopology(client, cluster, object); if (id !== requestId) return; topologyData = data; From f8f1a4ec39965852b723d69aedb0fe7f4c9052e3 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 19:46:39 +0800 Subject: [PATCH 33/41] fix(dashboard): show workspace picker only at cluster scope - render namespace picker when admin and cluster metrics both allowed - keep an admin with unknown access on their own workspace --- .../components/dashboard/compute/index.svelte | 16 ++++++++++------ src/lib/components/dashboard/model/index.svelte | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/lib/components/dashboard/compute/index.svelte b/src/lib/components/dashboard/compute/index.svelte index b106b765..f4f15502 100644 --- a/src/lib/components/dashboard/compute/index.svelte +++ b/src/lib/components/dashboard/compute/index.svelte @@ -126,12 +126,16 @@ {/if}
- + + {#if isClusterAdmin && canViewClusterMetrics} + + {/if} - + + {#if isClusterAdmin && canViewClusterMetrics} + + {/if} Date: Wed, 14 Oct 2026 19:46:47 +0800 Subject: [PATCH 34/41] fix(dashboard): batch gpu uuids by encoded length - size batches so gpu-fb-total stays under the hub's 6144-byte uri cap - count each uuid as the hub escapes and url-encodes it, five matchers - keep the 256-values cap for short uuids - test long, typical and short uuids against a replica of the hub query --- .../gpu-allocation/fetch-utilization.spec.ts | 65 ++++++++++++++++- .../gpu-allocation/fetch-utilization.ts | 69 +++++++++++++++++-- 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/src/lib/components/gpu-allocation/fetch-utilization.spec.ts b/src/lib/components/gpu-allocation/fetch-utilization.spec.ts index e53fbe57..aa8eac5e 100644 --- a/src/lib/components/gpu-allocation/fetch-utilization.spec.ts +++ b/src/lib/components/gpu-allocation/fetch-utilization.spec.ts @@ -3,7 +3,12 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import type { SharedMetrics, SharedParams, SharedQueryName } from '$lib/prometheus'; -import { fetchGpuAllocatedTotals, fetchGpuUtilization } from './fetch-utilization'; +import { + batchUuids, + encodedMatcherValueBytes, + fetchGpuAllocatedTotals, + fetchGpuUtilization +} from './fetch-utilization'; type Answers = Partial, number][]>>; type Call = { name: SharedQueryName; params?: SharedParams }; @@ -50,9 +55,9 @@ describe('fetchGpuUtilization', () => { expect(result.has('GPU-b')).toBe(false); }); - it('splits more than 256 cards into requests the server accepts', async () => { + it('splits more than 256 short cards into requests of at most 256', async () => { const calls: Call[] = []; - const ids = Array.from({ length: 300 }, (_, i) => `GPU-${i}`); + const ids = Array.from({ length: 300 }, (_, i) => `G${i}`); await fetchGpuUtilization( fakeShared({ 'gpu-util': [], 'gpu-fb-used': [], 'gpu-fb-total': [] }, calls), ids @@ -64,6 +69,20 @@ describe('fetchGpuUtilization', () => { expect(asked.sort()).toEqual([...ids].sort()); }); + it('splits long cards by length, so every request stays under the server limit', async () => { + const calls: Call[] = []; + // Dots and colons cost the most once escaped; 128 characters is the longest UUID allowed. + const ids = Array.from({ length: 300 }, (_, i) => `${i}:`.padEnd(128, '.')); + await fetchGpuUtilization( + fakeShared({ 'gpu-util': [], 'gpu-fb-used': [], 'gpu-fb-total': [] }, calls), + ids + ); + + const totals = calls.filter((c) => c.name === 'gpu-fb-total'); + expect(totals.length).toBeGreaterThan(1); + for (const c of totals) expect(serverUri(c.params?.uuid ?? []).length).toBeLessThan(6144 - 256); + expect(totals.flatMap((c) => c.params?.uuid ?? []).sort()).toEqual([...ids].sort()); + }); it('leaves cards unmeasured when the shared scope fails, without throwing', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); await expect(fetchGpuUtilization(fakeShared({}), ['GPU-a'])).resolves.toEqual(new Map()); @@ -92,3 +111,43 @@ describe('fetchGpuAllocatedTotals', () => { expect(warn).toHaveBeenCalled(); }); }); + +/** + * The request the server forwards for gpu-fb-total, built the way + * proxy_shared_catalog.go builds it: each value regexp-escaped, joined with + * `|`, quoted, put in all five matchers, then URL-encoded behind the agent's path. + */ +function serverUri(uuids: string[]): string { + const literal = uuids.map((u) => u.replace(/[\\.+*?()|[\]{}^$]/g, '\\$&')).join('|'); + const matcher = `{UUID=~${JSON.stringify(literal)}}`; + const query = + `sum by (UUID) (DCGM_FI_DEV_FB_USED${matcher} + DCGM_FI_DEV_FB_FREE${matcher} + DCGM_FI_DEV_FB_RESERVED${matcher})` + + ` or sum by (UUID) (DCGM_FI_DEV_FB_USED${matcher} + DCGM_FI_DEV_FB_FREE${matcher})`; + const search = new URLSearchParams({ query, time: '1760000000.123' }); + return `/__otterscale/proxy/api/v1/query?${search}`; +} + +describe('batchUuids', () => { + it('counts what escaping and URL encoding add', () => { + expect(encodedMatcherValueBytes('GPU-a_b')).toBe(7); + expect(encodedMatcherValueBytes('.')).toBe(7); + expect(encodedMatcherValueBytes(':')).toBe(3); + }); + + it('keeps a typical NVIDIA UUID batch under the server limit', () => { + const ids = Array.from( + { length: 1000 }, + (_, i) => `GPU-${String(i).padStart(8, '0')}-1234-1234-1234-123456789abc` + ); + const batches = batchUuids(ids); + for (const batch of batches) expect(serverUri(batch).length).toBeLessThan(6144 - 256); + expect(batches.flat()).toEqual(ids); + }); + + it('gives the longest UUID a request of its own when needed', () => { + const longest = '.'.repeat(128); + const batches = batchUuids([longest, longest]); + expect(batches).toEqual([[longest], [longest]]); + expect(serverUri([longest]).length).toBeLessThan(6144); + }); +}); diff --git a/src/lib/components/gpu-allocation/fetch-utilization.ts b/src/lib/components/gpu-allocation/fetch-utilization.ts index 07375c64..a3351707 100644 --- a/src/lib/components/gpu-allocation/fetch-utilization.ts +++ b/src/lib/components/gpu-allocation/fetch-utilization.ts @@ -23,6 +23,67 @@ const SAFE_UUID = /^[A-Za-z0-9_.:-]{1,128}$/; const MAX_UUIDS_PER_REQUEST = 256; const BYTES_PER_MIB = 1024 * 1024; +/** + * The server forwards a shared query as GET, with the PromQL it builds in the + * URL, and refuses with 400 a request whose path and encoded query together + * are longer than this (proxy_shared.go). + */ +const SERVER_URI_LIMIT = 6144; +/** + * What a request costs before any UUID: the agent's path, `query=` with the + * template, `&time=`. gpu-fb-total, the longest, measures 325 bytes with an + * empty matcher; this leaves room for a longer path or timestamp. + */ +const URI_OVERHEAD = 512; +/** Kept free below the limit, against a template that grows a little. */ +const URI_MARGIN = 512; +/** gpu-fb-total names the UUID matcher five times; gpu-util and gpu-fb-used once. */ +const UUID_MATCHERS_PER_QUERY = 5; +/** Bytes each matcher may hold: (6144 - 512 - 512) / 5 = 1024. */ +const MATCHER_BUDGET = Math.floor( + (SERVER_URI_LIMIT - URI_OVERHEAD - URI_MARGIN) / UUID_MATCHERS_PER_QUERY +); +/** The `|` that joins two values, as %7C. */ +const SEPARATOR_BYTES = 3; + +/** + * Bytes one value adds to a matcher once the server has escaped it the way + * sharedMatcher does — regexp.QuoteMeta, then a quoted PromQL string — and + * the query string is URL-encoded: `.` turns into `%5C%5C.`, `:` into `%3A`. + */ +export function encodedMatcherValueBytes(value: string): number { + const quoted = value.replace(/[\\.+*?()|[\]{}^$]/g, '\\$&').replace(/[\\"]/g, '\\$&'); + let bytes = 0; + for (const ch of quoted) bytes += /[A-Za-z0-9\-_.~]/.test(ch) ? 1 : 3; + return bytes; +} + +/** + * Split UUIDs into requests the server accepts: each matcher stays within + * MATCHER_BUDGET once encoded, so the longest query stays well under + * SERVER_URI_LIMIT, and no request names more than MAX_UUIDS_PER_REQUEST. + */ +export function batchUuids(uuids: string[]): string[][] { + const batches: string[][] = []; + let batch: string[] = []; + let bytes = 0; + for (const uuid of uuids) { + const cost = encodedMatcherValueBytes(uuid) + SEPARATOR_BYTES; + if ( + batch.length > 0 && + (bytes + cost > MATCHER_BUDGET || batch.length >= MAX_UUIDS_PER_REQUEST) + ) { + batches.push(batch); + batch = []; + bytes = 0; + } + batch.push(uuid); + bytes += cost; + } + if (batch.length > 0) batches.push(batch); + return batches; +} + /** One shared instant query, reduced to value-by-label. A failure yields an empty map, never throws. */ async function instantBy( shared: SharedMetrics, @@ -50,11 +111,9 @@ async function instantByUuid( name: SharedQueryName, uuids: string[] ): Promise> { - const chunks: string[][] = []; - for (let i = 0; i < uuids.length; i += MAX_UUIDS_PER_REQUEST) { - chunks.push(uuids.slice(i, i + MAX_UUIDS_PER_REQUEST)); - } - const maps = await Promise.all(chunks.map((uuid) => instantBy(shared, name, 'UUID', { uuid }))); + const maps = await Promise.all( + batchUuids(uuids).map((uuid) => instantBy(shared, name, 'UUID', { uuid })) + ); return new Map(maps.flatMap((map) => [...map])); } From f18754a929875bda655fe59644b79aa8929c098b Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 19:48:40 +0800 Subject: [PATCH 35/41] docs(dashboard): name kubevirt chart 1.2.0 in the guard test - the podmonitor label change ships in kubevirt 1.2.0, not 1.1.2 --- src/lib/components/dashboard/namespace-scope.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/dashboard/namespace-scope.spec.ts b/src/lib/components/dashboard/namespace-scope.spec.ts index 9dc3e41b..4ca63c90 100644 --- a/src/lib/components/dashboard/namespace-scope.spec.ts +++ b/src/lib/components/dashboard/namespace-scope.spec.ts @@ -11,7 +11,7 @@ function files(dir: string): string[] { } // Namespace scope forces namespace="". KubeVirt's PodMonitor keeps the -// VMI's own namespace label (helm-charts kubevirt 1.1.2), so a selector on +// VMI's own namespace label (helm-charts kubevirt 1.2.0), so a selector on // exported_namespace would only ever match nothing for a tenant. describe('compute dashboard', () => { it('selects VM metrics on namespace, not exported_namespace', () => { From 551ea8669e3d11b3f538b34ee1220c99bfe5216b Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 19:48:45 +0800 Subject: [PATCH 36/41] test(dashboard): scan every workspace page directory in the guard - scan workspace, model, compute, gpu-allocation, kind-viewer and routes - leave out the cluster and storage dashboards, admins only - forbid new PrometheusDriver and infra helpers from $lib/prometheus - check the listed helpers are still exported, so none goes stale - require actual shared-scope calls instead of the word shared --- .../dashboard/namespace-scope.spec.ts | 106 ++++++++++++++++-- 1 file changed, 95 insertions(+), 11 deletions(-) diff --git a/src/lib/components/dashboard/namespace-scope.spec.ts b/src/lib/components/dashboard/namespace-scope.spec.ts index 4ca63c90..0507d4a3 100644 --- a/src/lib/components/dashboard/namespace-scope.spec.ts +++ b/src/lib/components/dashboard/namespace-scope.spec.ts @@ -1,5 +1,5 @@ -import { readdirSync, readFileSync, statSync } from 'node:fs'; -import { join } from 'node:path'; +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -29,9 +29,95 @@ describe('compute dashboard', () => { }); }); -// GPU, KubeVirt and Ceph numbers on Workspace pages come from the server's -// named queries (shared scope), for admins and members alike: these files pick -// no cluster-scope driver and carry no infrastructure PromQL of their own. +/** + * Everything a Workspace page renders: its routes and the components they use. + * The cluster and storage dashboards live under the Workspace route too, but + * only viewers with cluster metrics access reach them. + */ +const workspaceRoots = [ + 'src/lib/components/dashboard/workspace', + 'src/lib/components/dashboard/model', + 'src/lib/components/dashboard/compute', + 'src/lib/components/gpu-allocation', + 'src/lib/components/kind-viewer', + 'src/routes/(auth)/[cluster]/[workspace]' +]; +const clusterOnly = [ + 'src/lib/components/dashboard/cluster/', + 'src/lib/components/dashboard/storage/', + 'src/routes/(auth)/[cluster]/[workspace]/dashboard/cluster/', + 'src/routes/(auth)/[cluster]/[workspace]/dashboard/storage/' +]; + +function workspaceFiles(): string[] { + return workspaceRoots + .flatMap((root) => files(join(process.cwd(), root))) + .map((f) => relative(process.cwd(), f).split(sep).join('/')) + .filter((f) => /\.(svelte|ts)$/.test(f) && !f.endsWith('.spec.ts')) + .filter((f) => !clusterOnly.some((dir) => f.startsWith(dir))); +} + +/** + * $lib/prometheus exports that reach past the Workspace's namespace: the + * cluster-scope driver, and PromQL pieces over node, GPU and disk exporters. + */ +const infrastructureHelpers = [ + 'clusterPrometheusDriver', + 'dcgmNodeSelector', + 'DCGM_GPU_MEMORY_TOTAL_BYTES', + 'DCGM_GPU_MEMORY_USED_BYTES', + 'ai100DiskRate', + 'ai100DiskRateByNode', + 'AIDAPTIV_CACHE_DEVICE_CLASS' +]; + +/** Infrastructure metrics: their series can name other tenants' pods, or none. */ +const infrastructureMetrics = + /DCGM_FI_|hami_|kubevirt_info|kubevirt_virt_controller|\bceph_|node_disk_|node_uname_info|topolvm_/; + +const forbidden: [string, RegExp][] = [ + ['new PrometheusDriver', /\bnew\s+PrometheusDriver\b/], + ...infrastructureHelpers.map((name): [string, RegExp] => [name, new RegExp(`\\b${name}\\b`)]), + ['infrastructure metric', infrastructureMetrics] +]; + +// Workspace pages make drivers with workspacePrometheusDriver, which keeps a +// viewer without cluster metrics access at namespace scope, and take GPU, +// KubeVirt and Ceph numbers from the server's named queries (shared scope), +// for admins and members alike. No file they render may pick the cluster +// scope or carry infrastructure PromQL of its own. +describe('workspace pages', () => { + const scanned = workspaceFiles(); + + it('scans every directory a Workspace page renders from', () => { + for (const root of workspaceRoots) expect(existsSync(join(process.cwd(), root))).toBe(true); + expect(scanned).toContain('src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte'); + expect(scanned).toContain( + 'src/routes/(auth)/[cluster]/[workspace]/dashboard/model/+page.svelte' + ); + expect(scanned.some((f) => clusterOnly.some((dir) => f.startsWith(dir)))).toBe(false); + }); + + it('forbids helpers that $lib/prometheus still exports', () => { + const source = readFileSync(join(process.cwd(), 'src/lib/prometheus.ts'), 'utf8'); + for (const name of infrastructureHelpers) { + expect(source).toMatch(new RegExp(`export (const|function) ${name}\\b`)); + } + }); + + it('use no cluster-scope driver and no infrastructure PromQL', () => { + const offenders = scanned.flatMap((file) => { + const source = readFileSync(join(process.cwd(), file), 'utf8'); + return forbidden + .filter(([, pattern]) => pattern.test(source)) + .map(([name]) => `${file}: ${name}`); + }); + expect(offenders).toEqual([]); + }); +}); + +// The places that show cluster-wide numbers on a Workspace page, and the +// shared-scope calls they make for them. describe('shared-scope components', () => { const sharedFiles = [ 'src/lib/components/gpu-allocation/gpu-allocation-dialog.svelte', @@ -43,15 +129,13 @@ describe('shared-scope components', () => { 'src/lib/components/dashboard/compute/overview/worker.svelte', 'src/lib/components/dashboard/workspace/overview/storage.svelte' ]; - const infrastructure = - /DCGM_FI_|hami_|kubevirt_info|kubevirt_virt_controller|ceph_cluster_total_bytes/; + const sharedCall = + /\bshared\.(instant|range)\(\s*'[a-z-]+'|\bfetchGpu(Utilization|AllocatedTotals)\(\s*shared\b/; for (const file of sharedFiles) { - it(`${file} uses the shared scope`, () => { + it(`${file} queries the shared scope`, () => { const source = readFileSync(join(process.cwd(), file), 'utf8'); - expect(source).not.toContain('clusterPrometheusDriver'); - expect(source).not.toMatch(infrastructure); - expect(source).toMatch(/\bshared\b/); + expect(source).toMatch(sharedCall); }); } }); From 2a47b273c6cea18e9d8c0c73dcb12d22a38cba67 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 19:52:47 +0800 Subject: [PATCH 37/41] fix(dashboard): show unknown, not zero, when a metric fails - add fetchCombinedInstantOrNull, null on failure, with a warning - grey out restart, oom, throttle, shm and preempt when it fails - warn when the gpu memory card fails to load - clear kubevirt cards on a failed reload and show no data - show an error in gpu resources instead of 0 devices - add pod_health_unavailable and gpu_resources_fetch_error messages --- messages/en.json | 2 + messages/zh-hant.json | 2 + .../compute/overview/controller.svelte | 5 +- .../dashboard/compute/overview/worker.svelte | 4 +- .../model/analytics/b-gpu-memory.svelte | 3 +- .../model/analytics/b-pod-health.svelte | 75 ++-- .../utils/gpu-resources.svelte | 338 +++++++++--------- src/lib/prometheus.spec.ts | 34 +- src/lib/prometheus.ts | 30 +- 9 files changed, 293 insertions(+), 200 deletions(-) diff --git a/messages/en.json b/messages/en.json index 063cbbe4..3d292779 100644 --- a/messages/en.json +++ b/messages/en.json @@ -176,6 +176,7 @@ "gpu_allocation_node_description": "Pod → GPU → Node relationship diagram", "gpu_allocation_loading": "Loading GPU allocation...", "gpu_allocation_fetch_error": "Failed to fetch GPU allocation: {message}", + "gpu_resources_fetch_error": "Couldn't load GPU resources: {message}", "gpu_allocation_empty_model": "No GPU devices found for this model. Pods may not have HAMi vGPU allocations.", "gpu_allocation_empty_node": "No GPU devices registered on this node.", "node_mig_enable_alert_title": "This feature is not recommended", @@ -642,6 +643,7 @@ "metric_oom": "OOM", "metric_xid": "XID", "xid_unavailable": "Couldn't load XID errors right now. Refresh to try again.", + "pod_health_unavailable": "Couldn't load pod health right now. Refresh to try again.", "metric_cpu_throttle": "Throttle", "metric_shm_crash": "Shm", "metric_preempt": "Preempt", diff --git a/messages/zh-hant.json b/messages/zh-hant.json index 6f8f1d49..d4b01db5 100644 --- a/messages/zh-hant.json +++ b/messages/zh-hant.json @@ -176,6 +176,7 @@ "gpu_allocation_node_description": "Pod → GPU → Node 關聯圖", "gpu_allocation_loading": "正在載入 GPU 分配資訊...", "gpu_allocation_fetch_error": "無法取得 GPU 分配資訊：{message}", + "gpu_resources_fetch_error": "無法取得 GPU 資源：{message}", "gpu_allocation_empty_model": "此模型未找到 GPU 裝置。Pod 可能沒有 HAMi vGPU 分配。", "gpu_allocation_empty_node": "此節點上沒有已註冊的 GPU 裝置。", "node_mig_enable_alert_title": "不建議使用此功能", @@ -642,6 +643,7 @@ "metric_oom": "OOM", "metric_xid": "XID", "xid_unavailable": "暫時無法取得 XID 錯誤，請重新整理再試一次。", + "pod_health_unavailable": "暫時無法取得 Pod 狀態，請重新整理再試一次。", "metric_cpu_throttle": "節流", "metric_shm_crash": "Shm", "metric_preempt": "搶占", diff --git a/src/lib/components/dashboard/compute/overview/controller.svelte b/src/lib/components/dashboard/compute/overview/controller.svelte index cbc08cf9..8c7523e5 100644 --- a/src/lib/components/dashboard/compute/overview/controller.svelte +++ b/src/lib/components/dashboard/compute/overview/controller.svelte @@ -30,7 +30,10 @@ try { await Promise.all([fetchReadyControllers(), fetchHealthControllers()]); } catch (error) { - console.error('Failed to fetch controller data:', error); + // Show no data rather than the last numbers, which may no longer hold. + console.warn('Failed to fetch controller data:', error); + readyControllers = undefined; + healthControllers = undefined; } } diff --git a/src/lib/components/dashboard/compute/overview/worker.svelte b/src/lib/components/dashboard/compute/overview/worker.svelte index 768b0a6c..c11cbb7f 100644 --- a/src/lib/components/dashboard/compute/overview/worker.svelte +++ b/src/lib/components/dashboard/compute/overview/worker.svelte @@ -23,7 +23,9 @@ try { await fetchWorkers(); } catch (error) { - console.error('Failed to fetch worker data:', error); + // Show no data rather than the last number, which may no longer hold. + console.warn('Failed to fetch worker data:', error); + workers = undefined; } } diff --git a/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte b/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte index 8614cc74..46031e16 100644 --- a/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte +++ b/src/lib/components/dashboard/model/analytics/b-gpu-memory.svelte @@ -166,7 +166,8 @@ data = Object.values(dateMap).sort( (a, b) => (a.date as Date).getTime() - (b.date as Date).getTime() ); - } catch { + } catch (error) { + console.warn('Failed to fetch GPU memory:', error); failed = true; data = []; hosts = []; diff --git a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte index db265f83..30bf4edc 100644 --- a/src/lib/components/dashboard/model/analytics/b-pod-health.svelte +++ b/src/lib/components/dashboard/model/analytics/b-pod-health.svelte @@ -12,7 +12,7 @@ import { classifyThreshold, escapePromqlStringLiteral, - fetchCombinedInstant, + fetchCombinedInstantOrNull, fetchModelNodes, type SharedMetrics, type ThresholdLevel, @@ -49,6 +49,8 @@ let terminatedByPod = $state>({}); let xidByGpu = $state([]); let xidUnavailable = $state(false); + /** The namespace-scope query failed: restart, OOM, throttle, SHM and preempt are unknown. */ + let mainUnavailable = $state(false); let nodeCount = $state(0); let isLoaded = $state(false); @@ -174,20 +176,24 @@ const xidPromise = nodesPromise.then(fetchXid); const [main, nodes, xid] = await Promise.all([ - fetchCombinedInstant(prometheusDriver, mainQueries), + fetchCombinedInstantOrNull(prometheusDriver, mainQueries), nodesPromise, xidPromise ]); - restartTotal = Math.round(scalarFromVectors(main.restartTotal)); - restartByPod = perPodFromVectors(main.restartPerPod); - oomTotal = Math.round(scalarFromVectors(main.oomTotal)); - oomByPod = perPodFromVectors(main.oomPerPod); - terminatedByPod = terminatedFromVectors(main.lastTermExit); - const throttleV = scalarFromVectors(main.throttle); + // A failed query clears the last numbers rather than keeping them, and the + // tiles show unknown, like XID, not a healthy 0. + mainUnavailable = main === null; + const slot = (name: keyof typeof mainQueries) => main?.[name] ?? []; + restartTotal = Math.round(scalarFromVectors(slot('restartTotal'))); + restartByPod = perPodFromVectors(slot('restartPerPod')); + oomTotal = Math.round(scalarFromVectors(slot('oomTotal'))); + oomByPod = perPodFromVectors(slot('oomPerPod')); + terminatedByPod = terminatedFromVectors(slot('lastTermExit')); + const throttleV = scalarFromVectors(slot('throttle')); throttlePct = Number.isFinite(throttleV) ? throttleV : 0; - preemptTotal = Math.round(scalarFromVectors(main.preemptTotal)); - preemptByPod = perPodFromVectors(main.preemptPerPod); + preemptTotal = Math.round(scalarFromVectors(slot('preemptTotal'))); + preemptByPod = perPodFromVectors(slot('preemptPerPod')); nodeCount = nodes?.length ?? 0; xidUnavailable = xid === null; xidTotal = xid?.total ?? 0; @@ -236,6 +242,11 @@ if (level === 'orange') return 'text-chart-1'; return 'text-chart-2'; } + + /** Colour of a tile fed by the namespace-scope query: grey while it is unknown. */ + function mainColorClass(level: ThresholdLevel): string { + return mainUnavailable ? 'text-muted-foreground' : valueColorClass(level); + } @@ -269,15 +280,19 @@
-

- {restartTotal} +

+ {mainUnavailable ? '—' : restartTotal}

{m.metric_restart()}

- {#if restartByPod.length > 0} + {#if mainUnavailable} + +

{m.pod_health_unavailable()}

+ + {:else if restartByPod.length > 0}
    {#each restartByPod as p (p.pod)} @@ -294,15 +309,19 @@
    -

    - {oomTotal} +

    + {mainUnavailable ? '—' : oomTotal}

    {m.metric_oom()}

    - {#if oomByPod.length > 0} + {#if mainUnavailable} + +

    {m.pod_health_unavailable()}

    +     + {:else if oomByPod.length > 0}
      {#each oomByPod as p (p.pod)} @@ -358,8 +377,8 @@
      -

      - {throttlePct.toFixed(1)}% +

      + {mainUnavailable ? '—' : `${throttlePct.toFixed(1)}%`}

      {m.metric_cpu_throttle()} @@ -367,15 +386,17 @@

      -

      {m.llm_dashboard_cpu_throttle_hint()}

      +

      + {mainUnavailable ? m.pod_health_unavailable() : m.llm_dashboard_cpu_throttle_hint()} +

      -

      - {shmCrashCount} +

      + {mainUnavailable ? '—' : shmCrashCount}

      {m.metric_shm_crash()} @@ -383,7 +404,9 @@

      - {#if shmCrashPods.length === 0} + {#if mainUnavailable} +

      {m.pod_health_unavailable()}

      + {:else if shmCrashPods.length === 0}

      {m.llm_dashboard_shm_crash_hint()}

      {:else}
      @@ -406,8 +429,8 @@
      -

      - {preemptTotal} +

      + {mainUnavailable ? '—' : preemptTotal}

      {m.metric_preempt()} @@ -415,7 +438,9 @@

      - {#if preemptByPod.length === 0} + {#if mainUnavailable} +

      {m.pod_health_unavailable()}

      + {:else if preemptByPod.length === 0}

      {m.llm_dashboard_preempt_hint()}

      {:else}
      diff --git a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte index 2d064b0f..bab55e85 100644 --- a/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte +++ b/src/lib/components/kind-viewer/kind-viewer-actions/utils/gpu-resources.svelte @@ -10,11 +10,14 @@ import * as Modal from '$lib/components/ui/dialog'; import * as Item from '$lib/components/ui/item'; import * as Table from '$lib/components/ui/table'; + import { m } from '$lib/messages'; import { sharedMetrics } from '$lib/prometheus'; let gpus: Record[] = []; let gpusByModelName: Record[] = []; let gpusByHostName: Record[] = []; + /** Set when the numbers could not be loaded, so an empty table is not read as no GPUs. */ + let errorMessage: string | null = null; onMount(async () => { // Per-card inventory and vGPU totals: cluster-wide, naming no workload, @@ -71,8 +74,13 @@ total: lodash.sumBy(value, 'total') })); } catch (error) { - // The tables stay empty. - console.error('Failed to fetch GPU resources:', error); + console.warn('Failed to fetch GPU resources:', error); + gpus = []; + gpusByModelName = []; + gpusByHostName = []; + errorMessage = m.gpu_resources_fetch_error({ + message: error instanceof Error ? error.message : String(error) + }); } }); @@ -90,172 +98,178 @@ GPU Resources Amount of limits, usage, and total field buffers. - {gpus.length} devices + {#if !errorMessage} + {gpus.length} devices + {/if} - - - Summary - - {gpusByModelName.length} types - - - - - - - Identifier - usage - limit - total - - - - {#each gpusByModelName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + {#if errorMessage} +

      {errorMessage}

      + {:else} + + + Summary + + {gpusByModelName.length} types + + + + + + + Identifier + usage + limit + total - {/each} -       -       + + + {#each gpusByModelName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + - - - Summary - - {gpusByHostName.length} hosts - - - - - - - Identifier - usage - limit - total - - - - {#each gpusByHostName as gpuByIdetifier, index (index)} - - {gpuByIdetifier.identifier} - - {@const { value, unit } = - gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' - ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + + + Summary + + {gpusByHostName.length} hosts + + + + + + + Identifier + usage + limit + total - {/each} - - + + + {#each gpusByHostName as gpuByIdetifier, index (index)} + + {gpuByIdetifier.identifier} + + {@const { value, unit } = + gpuByIdetifier.usage && typeof gpuByIdetifier.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.limit && typeof gpuByIdetifier.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpuByIdetifier.total && typeof gpuByIdetifier.total === 'number' + ? formatWithBinarySuffix(BigInt(gpuByIdetifier.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + - - - Details - - {gpus.length} devices - - - - - - - Model Name - Host Name - Device - UUID - usage - limit - total - - - - {#each gpus as gpu, index (index)} - - {gpu.modelName} - {gpu.hostname} - {gpu.device} - {gpu.UUID} - - {@const { value, unit } = - gpu.usage && typeof gpu.usage === 'number' - ? formatWithBinarySuffix(BigInt(gpu.usage)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.limit && typeof gpu.limit === 'number' - ? formatWithBinarySuffix(BigInt(gpu.limit)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - - - {@const { value, unit } = - gpu.total && typeof gpu.total === 'number' - ? formatWithBinarySuffix(BigInt(gpu.total)) - : { value: undefined, unit: undefined }} - {#if value !== undefined && unit !== undefined} - {value.toFixed(2)} {unit} - {/if} - + + + Details + + {gpus.length} devices + + + + + + + Model Name + Host Name + Device + UUID + usage + limit + total - {/each} - - + + + {#each gpus as gpu, index (index)} + + {gpu.modelName} + {gpu.hostname} + {gpu.device} + {gpu.UUID} + + {@const { value, unit } = + gpu.usage && typeof gpu.usage === 'number' + ? formatWithBinarySuffix(BigInt(gpu.usage)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.limit && typeof gpu.limit === 'number' + ? formatWithBinarySuffix(BigInt(gpu.limit)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {@const { value, unit } = + gpu.total && typeof gpu.total === 'number' + ? formatWithBinarySuffix(BigInt(gpu.total)) + : { value: undefined, unit: undefined }} + {#if value !== undefined && unit !== undefined} + {value.toFixed(2)} {unit} + {/if} + + + {/each} + + + {/if} diff --git a/src/lib/prometheus.spec.ts b/src/lib/prometheus.spec.ts index ed657f82..0f91bfe5 100644 --- a/src/lib/prometheus.spec.ts +++ b/src/lib/prometheus.spec.ts @@ -2,12 +2,14 @@ import { createServer, type IncomingMessage, type Server } from 'node:http'; import type { AddressInfo } from 'node:net'; import type { PrometheusDriver } from 'prometheus-query'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { classifyGpuGovernance, clusterPrometheusDriver, dashboardBody, + fetchCombinedInstant, + fetchCombinedInstantOrNull, fetchModelNodes, namespacePrometheusDriver, namespaceSelector, @@ -227,3 +229,33 @@ describe('fetchModelNodes', () => { expect(nodes?.slice().sort()).toEqual(['n1', 'n2']); }); }); + +describe('fetchCombinedInstantOrNull', () => { + const driver = (instantQuery: () => Promise) => + ({ instantQuery }) as unknown as PrometheusDriver; + const queries = { restarts: 'a', ooms: 'b' }; + + it('resolves null when the query fails, so callers can show unknown rather than 0', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const failing = driver(async () => { + throw new Error('HTTP 403'); + }); + await expect(fetchCombinedInstantOrNull(failing, queries)).resolves.toBeNull(); + expect(warn).toHaveBeenCalled(); + await expect(fetchCombinedInstant(failing, queries)).resolves.toEqual({ + restarts: [], + ooms: [] + }); + warn.mockRestore(); + }); + + it('splits the combined answer by name, without the tag', async () => { + const answering = driver(async () => ({ + resultType: 'vector', + result: [{ metric: { labels: { __series__: 'ooms', pod: 'p1' } } }] + })); + const result = await fetchCombinedInstantOrNull(answering, queries); + expect(result?.restarts).toEqual([]); + expect(result?.ooms.map((v) => v.metric.labels)).toEqual([{ pod: 'p1' }]); + }); +}); diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index d7c827c3..94469eba 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -582,16 +582,13 @@ export async function fetchCombinedFlattenedRange( /** * Run multiple named instant queries as a SINGLE HTTP request. * Returns a record mapping each query name to its InstantVector list (after the - * `__series__` tag has been stripped, so callers see the original label sets). - * - * On error, every named slot is set to `[]` so callers can deconstruct without - * needing a try/catch per query — matching the lenient `instantNumber` / - * `instantPerPod` pattern used in the dashboard. + * `__series__` tag has been stripped, so callers see the original label sets), + * or null when the request failed, so callers can show "unknown" rather than 0. */ -export async function fetchCombinedInstant( +export async function fetchCombinedInstantOrNull( client: PrometheusDriver, queries: Record -): Promise> { +): Promise | null> { const result: Record = {}; for (const name of Object.keys(queries)) result[name] = []; const combined = Object.entries(queries) @@ -608,12 +605,27 @@ export async function fetchCombinedInstant( vector.metric.labels = rest; result[name].push(vector); } - } catch { - // leave all slots empty + } catch (error) { + console.warn('Failed to run combined instant query:', error); + return null; } return result; } +/** + * {@link fetchCombinedInstantOrNull}, with every named slot set to `[]` on + * error so callers can deconstruct without needing a try/catch per query — + * matching the lenient `instantNumber` / `instantPerPod` pattern used in the + * dashboard. + */ +export async function fetchCombinedInstant( + client: PrometheusDriver, + queries: Record +): Promise> { + const result = await fetchCombinedInstantOrNull(client, queries); + return result ?? Object.fromEntries(Object.keys(queries).map((name) => [name, []])); +} + /** * Label selector scoping DCGM (GPU exporter) series to one Kubernetes node. DCGM's * `hostname` label equals the K8s node name (as the model dashboard's GPU panels also assume). From 91845498d9faa830633a3c68fe5d2787b8d27e63 Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 19:52:52 +0800 Subject: [PATCH 38/41] docs(dashboard): say shared requests may omit the namespace - carry the workspace's namespace when there is one, omit it otherwise --- docs/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metrics.md b/docs/metrics.md index 786a22f1..8fd69f25 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -8,7 +8,7 @@ The OtterScale server answers metrics at three scopes; the shared one takes no P | Namespace | `/proxy//prometheus-tenancy?namespace=` | anyone who may `get pods` in `` | any query, rewritten to `namespace=""` the way prom-label-proxy does it | | Shared | `/proxy//prometheus-shared/` | the namespace scope's callers, and cluster scope | one of the server's named queries: per GPU card, per host, or a total; no PromQL | -Everyone else gets 403 at cluster scope, whatever the query. Create drivers with `clusterPrometheusDriver`, `namespacePrometheusDriver` or `workspacePrometheusDriver` (`src/lib/prometheus.ts`), never `new PrometheusDriver` with a hand-written endpoint. `page.data.canViewClusterMetrics` says which scope the viewer has. Use `sharedMetrics(cluster, namespace)` for the shared scope; `namespace` is always the Workspace's. +Everyone else gets 403 at cluster scope, whatever the query. Create drivers with `clusterPrometheusDriver`, `namespacePrometheusDriver` or `workspacePrometheusDriver` (`src/lib/prometheus.ts`), never `new PrometheusDriver` with a hand-written endpoint. `page.data.canViewClusterMetrics` says which scope the viewer has. Use `sharedMetrics(cluster, namespace)` for the shared scope; `namespace` is the Workspace's when there is one, never an admin's namespace choice, and is left out otherwise, which only cluster viewers may do. ## Rules for namespace-scoped pages From 4c157c28d093f75b6df9ec645fe226906ac7245b Mon Sep 17 00:00:00 2001 From: hstsai Date: Wed, 14 Oct 2026 19:55:57 +0800 Subject: [PATCH 39/41] test(dashboard): check the hub's own 403s in the tenant e2e - ask cluster scope, another namespace and shared without one directly - expect 403 for each, and 200 for shared as the user's own workspace - wait for the shared answers each step sends, registered before it - poll for chart data instead of a fixed three-second sleep --- e2e/tenant-metrics.test.ts | 107 ++++++++++++++++++++++++++++--------- 1 file changed, 82 insertions(+), 25 deletions(-) diff --git a/e2e/tenant-metrics.test.ts b/e2e/tenant-metrics.test.ts index 4b3bbf97..51632bae 100644 --- a/e2e/tenant-metrics.test.ts +++ b/e2e/tenant-metrics.test.ts @@ -14,7 +14,8 @@ * scope. E2E_EXPECT_VM_DATA / E2E_EXPECT_MODEL_DATA=1 also require data on the * compute / model pages (set them when the namespace runs a VM / a model); * E2E_MODEL names the model for the analytics and model-service tests, which - * are skipped without it. + * are skipped without it. E2E_OTHER_NAMESPACE (default kube-system) is a + * namespace the user has no access to, for the server's own refusals. */ import { expect, type Page, test } from '@playwright/test'; @@ -29,6 +30,9 @@ const env = { /** Optional: the model whose analytics and actions are checked. */ const model = process.env.E2E_MODEL ?? ''; +/** A namespace the user may not see, for asking the server what the dashboard never asks. */ +const otherNamespace = process.env.E2E_OTHER_NAMESPACE ?? 'kube-system'; + /** The server's named queries (otterscale internal/handler/proxy_shared_catalog.go). */ const SHARED_NAMES = new Set([ 'gpu-util', @@ -107,6 +111,17 @@ function watchMetrics(page: Page) { return { tenancy, shared, clusterScope }; } +/** + * Wait for an answer to each named shared query. Start it before the action + * that sends them, so the answers it waits for come after that action. + */ +function sharedAnswers(page: Page, names: string[]) { + const path = (name: string) => `/proxy/${env.cluster}/prometheus-shared/${name}`; + return Promise.all( + names.map((name) => page.waitForResponse((res) => new URL(res.url()).pathname === path(name))) + ); +} + /** Every shared call is a known name, answered, asked as this Workspace; `names` were all asked. */ function expectShared(calls: SharedCall[], names: string[]) { for (const c of calls) { @@ -162,11 +177,19 @@ async function login(page: Page) { const dashboard = (name: string) => `/${env.cluster}/${env.workspace}/dashboard/${name}`; -async function visit(page: Page, name: string) { +/** + * Open a dashboard and wait until its widgets, which query after mount, have + * been answered: at least one namespace-scoped query, and each of `sharedNames`. + */ +async function visit(page: Page, name: string, sharedNames: string[] = []) { const watched = watchMetrics(page); + const answered = sharedAnswers(page, sharedNames); await page.goto(dashboard(name)); + await answered; + await expect + .poll(() => watched.tenancy.length, { message: 'no namespace-scoped query' }) + .toBeGreaterThan(0); await page.waitForLoadState('networkidle'); - await page.waitForTimeout(3_000); // widgets query after mount return watched; } @@ -197,28 +220,54 @@ for (const name of ['cluster', 'storage']) { }); } +test('the server refuses cluster scope, another namespace, and shared without a namespace', async ({ + page +}) => { + expect(otherNamespace).not.toBe(env.namespace); + // Straight to the server through the dashboard's proxy, as the page's own requests go. + const ask = (path: string, params: Record = {}) => + page.request.get(`/proxy/${env.cluster}/${path}`, { + headers: { 'x-proxy-target': 'api' }, + params + }); + + const cluster = await ask('prometheus/api/v1/query', { query: 'up' }); + expect(cluster.status(), 'cluster scope').toBe(403); + const tenancy = await ask('prometheus-tenancy/api/v1/query', { + query: 'up', + namespace: otherNamespace + }); + expect(tenancy.status(), `namespace scope in ${otherNamespace}`).toBe(403); + const shared = await ask('prometheus-shared/ceph-total-bytes'); + expect(shared.status(), 'shared scope without a namespace').toBe(403); + // The same shared query as this Workspace is answered: the 403 above is the namespace's doing. + const own = await ask('prometheus-shared/ceph-total-bytes', { namespace: env.namespace }); + expect(own.status(), 'shared scope as this Workspace').toBe(200); +}); + test('overview shows their namespace, and the cluster capacity through the shared scope', async ({ page }) => { - const { tenancy, shared, clusterScope } = await visit(page, 'overview'); + const { tenancy, shared, clusterScope } = await visit(page, 'overview', ['ceph-total-bytes']); expectOwnNamespaceOnly(tenancy); - expect(hasData(tenancy), 'overview charts are empty').toBe(true); + await expect.poll(() => hasData(tenancy), { message: 'overview charts are empty' }).toBe(true); expect(clusterScope).toEqual([]); for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); expectShared(shared, ['ceph-total-bytes']); }); test('compute stays in their namespace and shows KubeVirt components', async ({ page }) => { - const { tenancy, shared, clusterScope } = await visit(page, 'compute'); + const kubevirt = ['kubevirt-handlers', 'kubevirt-controllers', 'kubevirt-controllers-ready']; + const { tenancy, shared, clusterScope } = await visit(page, 'compute', kubevirt); expectOwnNamespaceOnly(tenancy); expect(clusterScope).toEqual([]); for (const q of sentQueries(tenancy)) { for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); expect(q).not.toContain('exported_namespace'); } - expectShared(shared, ['kubevirt-handlers', 'kubevirt-controllers', 'kubevirt-controllers-ready']); + expectShared(shared, kubevirt); if (process.env.E2E_EXPECT_VM_DATA === '1') - expect(hasData(tenancy), 'compute charts are empty').toBe(true); + await expect.poll(() => hasData(tenancy), { message: 'compute charts are empty' }).toBe(true); }); test('model overview stays in their namespace', async ({ page }) => { @@ -228,18 +277,27 @@ test('model overview stays in their namespace', async ({ page }) => { for (const q of sentQueries(tenancy)) for (const m of INFRASTRUCTURE) expect(q).not.toContain(m); expectShared(shared, []); if (process.env.E2E_EXPECT_MODEL_DATA === '1') - expect(hasData(tenancy), 'model charts are empty').toBe(true); + await expect.poll(() => hasData(tenancy), { message: 'model charts are empty' }).toBe(true); }); test('model analytics shows GPU memory and XID through the shared scope', async ({ page }) => { test.skip(!model, 'needs E2E_MODEL'); const watched = watchMetrics(page); + const gpuNames = [ + 'host-gpu-fb-used', + 'host-gpu-fb-total', + 'host-gpu-count', + 'gpu-xid-total', + 'gpu-xid-by-gpu' + ]; + // The panels query once a model is picked; started before any click, this + // also catches answers for a model the page picked on its own. + const answered = sharedAnswers(page, gpuNames); await page.goto(dashboard('model')); await page.getByRole('tab', { name: /^(Analytics|分析)$/ }).click(); await page.getByRole('combobox').first().click(); await page.getByRole('option', { name: model, exact: true }).click(); - await page.waitForLoadState('networkidle'); - await page.waitForTimeout(3_000); // panels query after the model is picked + await answered; await expect( page.getByText(/GPU Memory: Allocated vs Used|GPU 記憶體：已分配 vs 已使用/) @@ -247,13 +305,7 @@ test('model analytics shows GPU memory and XID through the shared scope', async await expect(page.getByText('XID', { exact: true })).toBeVisible(); expectOwnNamespaceOnly(watched.tenancy); expect(watched.clusterScope).toEqual([]); - expectShared(watched.shared, [ - 'host-gpu-fb-used', - 'host-gpu-fb-total', - 'host-gpu-count', - 'gpu-xid-total', - 'gpu-xid-by-gpu' - ]); + expectShared(watched.shared, gpuNames); }); test('a model service shows its GPU allocation and the GPU resources', async ({ page }) => { @@ -270,24 +322,29 @@ test('a model service shows its GPU allocation and the GPU resources', async ({ await page.goto(list); await openActions(); - await page.getByRole('menuitem', { name: /^(GPU Allocation|GPU 分配)$/ }).click(); - await expect(page.getByRole('dialog')).toBeVisible(); - await page.waitForLoadState('networkidle'); - expectShared(watched.shared, [ + const allocationNames = [ 'gpu-util', 'gpu-fb-used', 'gpu-fb-total', 'gpu-allocated-memory', 'gpu-shared-count' - ]); + ]; + const allocationAnswered = sharedAnswers(page, allocationNames); + await page.getByRole('menuitem', { name: /^(GPU Allocation|GPU 分配)$/ }).click(); + await expect(page.getByRole('dialog')).toBeVisible(); + await allocationAnswered; + expectShared(watched.shared, allocationNames); await page.keyboard.press('Escape'); await openActions(); + // The GPU resources ask as the Update dialog opens, before their button is clicked. + const resourceNames = ['gpu-inventory', 'vgpu-memory-limit', 'vgpu-memory-used']; + const resourcesAnswered = sharedAnswers(page, resourceNames); await page.getByRole('menuitem', { name: /^Update$/ }).click(); // The row behind the dialog has GPU icons too; the overlay would take the click. await page.getByRole('dialog').last().locator('button:has(svg.lucide-gpu)').first().click(); await expect(page.getByText('GPU Resources', { exact: true })).toBeVisible(); - await page.waitForLoadState('networkidle'); - expectShared(watched.shared, ['gpu-inventory', 'vgpu-memory-limit', 'vgpu-memory-used']); + await resourcesAnswered; + expectShared(watched.shared, resourceNames); expect(watched.clusterScope).toEqual([]); }); From 7dfc0d4fb543a1bd922dba79ab59a83932f26859 Mon Sep 17 00:00:00 2001 From: hstsai Date: Thu, 15 Oct 2026 10:29:29 +0800 Subject: [PATCH 40/41] fix(dashboard): count escaped uuid length per character - the chained replace read as double escaping to code scanning - same byte counts; tests pin backslash, quote and metacharacters --- .../gpu-allocation/fetch-utilization.spec.ts | 5 +++++ .../gpu-allocation/fetch-utilization.ts | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/lib/components/gpu-allocation/fetch-utilization.spec.ts b/src/lib/components/gpu-allocation/fetch-utilization.spec.ts index aa8eac5e..967449bd 100644 --- a/src/lib/components/gpu-allocation/fetch-utilization.spec.ts +++ b/src/lib/components/gpu-allocation/fetch-utilization.spec.ts @@ -132,6 +132,11 @@ describe('batchUuids', () => { expect(encodedMatcherValueBytes('GPU-a_b')).toBe(7); expect(encodedMatcherValueBytes('.')).toBe(7); expect(encodedMatcherValueBytes(':')).toBe(3); + // QuoteMeta then PromQL quoting: a backslash becomes four, a quote \" and + // a metacharacter \\ before it; each backslash, quote and + is %XX. + expect(encodedMatcherValueBytes('\\')).toBe(12); + expect(encodedMatcherValueBytes('"')).toBe(6); + expect(encodedMatcherValueBytes('a+b')).toBe(11); }); it('keeps a typical NVIDIA UUID batch under the server limit', () => { diff --git a/src/lib/components/gpu-allocation/fetch-utilization.ts b/src/lib/components/gpu-allocation/fetch-utilization.ts index a3351707..f28bd74e 100644 --- a/src/lib/components/gpu-allocation/fetch-utilization.ts +++ b/src/lib/components/gpu-allocation/fetch-utilization.ts @@ -46,15 +46,26 @@ const MATCHER_BUDGET = Math.floor( /** The `|` that joins two values, as %7C. */ const SEPARATOR_BYTES = 3; +/** The characters regexp.QuoteMeta puts a backslash before. */ +const REGEXP_META = /[\\.+*?()|[\]{}^$]/; + /** * Bytes one value adds to a matcher once the server has escaped it the way * sharedMatcher does — regexp.QuoteMeta, then a quoted PromQL string — and * the query string is URL-encoded: `.` turns into `%5C%5C.`, `:` into `%3A`. + * Only the length is computed, character by character; nothing is escaped. */ export function encodedMatcherValueBytes(value: string): number { - const quoted = value.replace(/[\\.+*?()|[\]{}^$]/g, '\\$&').replace(/[\\"]/g, '\\$&'); let bytes = 0; - for (const ch of quoted) bytes += /[A-Za-z0-9\-_.~]/.test(ch) ? 1 : 3; + for (const ch of value) { + // QuoteMeta adds one backslash before a metacharacter (a backslash + // becomes two); the PromQL string then doubles every backslash and + // puts one before a double quote. + let escaped = ch; + if (REGEXP_META.test(ch)) escaped = ch === '\\' ? '\\\\\\\\' : '\\\\' + ch; + else if (ch === '"') escaped = '\\"'; + for (const c of escaped) bytes += /[A-Za-z0-9\-_.~]/.test(c) ? 1 : 3; + } return bytes; } From 32a10ac57db393ace69f4957c6cf6f2360ff8495 Mon Sep 17 00:00:00 2001 From: hstsai Date: Thu, 15 Oct 2026 10:40:52 +0800 Subject: [PATCH 41/41] fix(deps): override vulnerable build-time packages - source-map-js 1.2.2 and postcss-selector-parser 7.1.6 - static-copy watches with chokidar 4, which drops unpatched braces - pnpm audit: no known vulnerabilities --- package.json | 5 +- pnpm-lock.yaml | 122 ++++++------------------------------------------- 2 files changed, 18 insertions(+), 109 deletions(-) diff --git a/package.json b/package.json index 991f776c..02178439 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,10 @@ "brace-expansion@<5.0.12": "5.0.12", "marked": "^18.0.7", "devalue@<5.9.3": "5.9.3", - "postcss@<8.5.28": "8.5.28" + "postcss@<8.5.28": "8.5.28", + "source-map-js@<1.2.2": "1.2.2", + "postcss-selector-parser@<7.1.6": "7.1.6", + "vite-plugin-static-copy>chokidar": "^4.0.3" }, "onlyBuiltDependencies": [ "esbuild", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 91f5fbb5..66115e0a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,9 @@ overrides: marked: ^18.0.7 devalue@<5.9.3: 5.9.3 postcss@<8.5.28: 8.5.28 + source-map-js@<1.2.2: 1.2.2 + postcss-selector-parser@<7.1.6: 7.1.6 + vite-plugin-static-copy>chokidar: ^4.0.3 importers: @@ -1503,10 +1506,6 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} - anymatch@3.1.3: - resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} - engines: {node: '>= 8'} - arctic@3.7.0: resolution: {integrity: sha512-ZMQ+f6VazDgUJOd+qNV+H7GohNSYal1mVjm5kEaZfE2Ifb7Ss70w+Q7xpJC87qZDkMZIXYf0pTIYZA0OPasSbw==} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. @@ -1545,10 +1544,6 @@ packages: bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} - binary-extensions@2.3.0: - resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} - engines: {node: '>=8'} - bits-ui@2.18.1: resolution: {integrity: sha512-KkemzKFH4T3gt3H+P86JcnAWExjByv/6vlwjm/BoCwTPHu03yiCdxbghdJLvFReQTe0acCAiRcKfmixxD6XvlA==} engines: {node: '>=20'} @@ -1560,10 +1555,6 @@ packages: resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} engines: {node: 20 || >=22} - braces@3.0.3: - resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} - engines: {node: '>=8'} - call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -1584,10 +1575,6 @@ packages: character-entities@2.0.2: resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} - chokidar@3.6.0: - resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} - engines: {node: '>= 8.10.0'} - chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -2036,10 +2023,6 @@ packages: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} - fill-range@7.1.1: - resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} - engines: {node: '>=8'} - find-up@5.0.0: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} @@ -2085,10 +2068,6 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} - glob-parent@5.1.2: - resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} - engines: {node: '>= 6'} - glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -2181,10 +2160,6 @@ packages: resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==} engines: {node: '>=12.22.0'} - is-binary-path@2.1.0: - resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} - engines: {node: '>=8'} - is-core-module@2.16.1: resolution: {integrity: sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==} engines: {node: '>= 0.4'} @@ -2200,10 +2175,6 @@ packages: is-module@1.0.0: resolution: {integrity: sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==} - is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} @@ -2606,10 +2577,6 @@ packages: resolution: {integrity: sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==} hasBin: true - normalize-path@3.0.0: - resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} - engines: {node: '>=0.10.0'} - obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} @@ -2660,10 +2627,6 @@ packages: picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - picomatch@2.3.2: - resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} - engines: {node: '>=8.6'} - picomatch@4.0.4: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} @@ -2710,8 +2673,8 @@ packages: peerDependencies: postcss: 8.5.28 - postcss-selector-parser@7.1.4: - resolution: {integrity: sha512-HeP7D2wyhkR+XaK6v4W8oRF62Dsz4flyuczALJp61GckGm42u1saSSJ/0auvcBqxs3jMRFEcPK34At/0JBKdOg==} + postcss-selector-parser@7.1.6: + resolution: {integrity: sha512-7qASPzhKF2l2KLboRZux8CCTRMdGiV08vWmyKzPz22qZ7ZjQBOeY7rNzNoCLSUiftJ7HUq0GERHmxw/t0dCdMw==} engines: {node: '>=4'} postcss@8.5.28: @@ -2817,10 +2780,6 @@ packages: resolution: {integrity: sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==} engines: {node: '>=0.12'} - readdirp@3.6.0: - resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} - engines: {node: '>=8.10.0'} - readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -2960,8 +2919,8 @@ packages: resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==} engines: {node: '>=18'} - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + source-map-js@1.2.2: + resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} engines: {node: '>=0.10.0'} space-separated-tokens@2.0.2: @@ -3104,10 +3063,6 @@ packages: resolution: {integrity: sha512-JIXCerhudr/N6OWLwLF1HVsTTUo7ry6qHa5eWZEkiMuxsIiAACL55tGLfqfHfoH7QaMQUW8fngD7u7TxWexYQg==} hasBin: true - to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} - totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -4056,7 +4011,7 @@ snapshots: jiti: 2.7.0 lightningcss: 1.32.0 magic-string: 0.30.21 - source-map-js: 1.2.1 + source-map-js: 1.2.2 tailwindcss: 4.3.3 '@tailwindcss/oxide-android-arm64@4.3.3': @@ -4446,11 +4401,6 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 - anymatch@3.1.3: - dependencies: - normalize-path: 3.0.0 - picomatch: 2.3.2 - arctic@3.7.0: dependencies: '@oslojs/crypto': 1.0.1 @@ -4487,8 +4437,6 @@ snapshots: dependencies: require-from-string: 2.0.2 - binary-extensions@2.3.0: {} - bits-ui@2.18.1(@internationalized/date@3.12.2)(@sveltejs/kit@2.70.3(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.56.4(@typescript-eslint/types@8.62.0))(vite@7.3.5(@types/node@24.13.4)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)))(svelte@5.56.4(@typescript-eslint/types@8.62.0))(typescript@5.9.3)(vite@7.3.5(@types/node@24.13.4)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)))(svelte@5.56.4(@typescript-eslint/types@8.62.0)): dependencies: '@floating-ui/core': 1.7.5 @@ -4506,10 +4454,6 @@ snapshots: dependencies: balanced-match: 4.0.4 - braces@3.0.3: - dependencies: - fill-range: 7.1.1 - call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -4525,18 +4469,6 @@ snapshots: character-entities@2.0.2: {} - chokidar@3.6.0: - dependencies: - anymatch: 3.1.3 - braces: 3.0.3 - glob-parent: 5.1.2 - is-binary-path: 2.1.0 - is-glob: 4.0.3 - normalize-path: 3.0.0 - readdirp: 3.6.0 - optionalDependencies: - fsevents: 2.3.3 - chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -4581,7 +4513,7 @@ snapshots: css-tree@3.2.1: dependencies: mdn-data: 2.27.1 - source-map-js: 1.2.1 + source-map-js: 1.2.2 cssesc@3.0.0: {} @@ -4978,10 +4910,6 @@ snapshots: dependencies: flat-cache: 4.0.1 - fill-range@7.1.1: - dependencies: - to-regex-range: 5.0.1 - find-up@5.0.0: dependencies: locate-path: 6.0.0 @@ -5030,10 +4958,6 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 - glob-parent@5.1.2: - dependencies: - is-glob: 4.0.3 - glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -5132,10 +5056,6 @@ snapshots: transitivePeerDependencies: - supports-color - is-binary-path@2.1.0: - dependencies: - binary-extensions: 2.3.0 - is-core-module@2.16.1: dependencies: hasown: 2.0.4 @@ -5148,8 +5068,6 @@ snapshots: is-module@1.0.0: {} - is-number@7.0.0: {} - is-plain-obj@4.1.0: {} is-potential-custom-element-name@1.0.1: {} @@ -5731,8 +5649,6 @@ snapshots: railroad-diagrams: 1.0.0 randexp: 0.4.6 - normalize-path@3.0.0: {} - obug@2.1.1: {} oniguruma-parser@0.12.2: {} @@ -5782,8 +5698,6 @@ snapshots: picocolors@1.1.1: {} - picomatch@2.3.2: {} - picomatch@4.0.4: {} playwright-core@1.61.1: {} @@ -5813,7 +5727,7 @@ snapshots: dependencies: postcss: 8.5.28 - postcss-selector-parser@7.1.4: + postcss-selector-parser@7.1.6: dependencies: cssesc: 3.0.0 util-deprecate: 1.0.2 @@ -5822,7 +5736,7 @@ snapshots: dependencies: nanoid: 3.3.19 picocolors: 1.1.1 - source-map-js: 1.2.1 + source-map-js: 1.2.2 prelude-ls@1.2.1: {} @@ -5863,10 +5777,6 @@ snapshots: discontinuous-range: 1.0.0 ret: 0.1.15 - readdirp@3.6.0: - dependencies: - picomatch: 2.3.2 - readdirp@4.1.2: {} redis-errors@1.2.0: {} @@ -6050,7 +5960,7 @@ snapshots: mrmime: 2.0.1 totalist: 3.0.1 - source-map-js@1.2.1: {} + source-map-js@1.2.2: {} space-separated-tokens@2.0.2: {} @@ -6098,7 +6008,7 @@ snapshots: espree: 10.4.0 postcss: 8.5.28 postcss-scss: 4.0.9(postcss@8.5.28) - postcss-selector-parser: 7.1.4 + postcss-selector-parser: 7.1.6 semver: 7.8.5 optionalDependencies: svelte: 5.56.4(@typescript-eslint/types@8.62.0) @@ -6209,10 +6119,6 @@ snapshots: dependencies: tldts-core: 7.0.29 - to-regex-range@5.0.1: - dependencies: - is-number: 7.0.0 - totalist@3.0.1: {} tough-cookie@6.0.1: @@ -6354,7 +6260,7 @@ snapshots: vite-plugin-static-copy@3.4.0(vite@7.3.5(@types/node@24.13.4)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)): dependencies: - chokidar: 3.6.0 + chokidar: 4.0.3 p-map: 7.0.4 picocolors: 1.1.1 tinyglobby: 0.2.16