From 966cf020e84ce39245344e676d19432224d24ca2 Mon Sep 17 00:00:00 2001 From: Likang Yu Date: Wed, 4 Nov 2026 17:19:06 +0800 Subject: [PATCH] feat: sync internal release-261104 Apply the source changes from internal tag release-261104 on top of the public release-261023 baseline. Source commit: c7478a2e20cede68123b17bc87f6ccf05eada047 Signed-off-by: Likang Yu --- api/config/crd/bases/rlinf.io_clusters.yaml | 6 + .../rlark.io/v1alpha1/clusterspec.go | 24 +- api/rlark.io/v1alpha1/annotations.go | 5 + api/rlark.io/v1alpha1/cluster_types.go | 12 +- apps/rlark-ui/src/App.tsx | 31 +- apps/rlark-ui/src/admin/Addons.tsx | 22 +- apps/rlark-ui/src/admin/AdminApp.tsx | 11 +- apps/rlark-ui/src/admin/AdminPage.tsx | 4 +- apps/rlark-ui/src/admin/CreateCluster.tsx | 22 +- apps/rlark-ui/src/backend.ts | 43 +- .../src/components/NodeResourceBrowser.tsx | 2 +- .../src/components/OverviewChinaMap.tsx | 6 +- .../components/ResourcePlacementPicker.tsx | 158 +++-- apps/rlark-ui/src/components/TagEditor.tsx | 3 +- .../src/components/TagFilterPopover.tsx | 123 ++-- apps/rlark-ui/src/components/terminal.tsx | 109 +++- apps/rlark-ui/src/crdModels.ts | 7 + apps/rlark-ui/src/data.ts | 43 +- .../rlark/_schemas/RlinfIoV1alpha1Cluster.js | 148 +++-- .../rlark/rlinf.io/v1alpha1/Cluster.ts | 4 +- apps/rlark-ui/src/i18n.ts | 8 +- apps/rlark-ui/src/mockBackend.ts | 6 +- apps/rlark-ui/src/pages/Api.tsx | 6 +- apps/rlark-ui/src/pages/ClusterManagement.tsx | 565 +++++++++++++---- apps/rlark-ui/src/pages/Clusters.tsx | 56 +- apps/rlark-ui/src/pages/CreateJob.tsx | 500 ++++++++------- apps/rlark-ui/src/pages/Domains.tsx | 12 +- apps/rlark-ui/src/pages/ImageRegistries.tsx | 16 +- apps/rlark-ui/src/pages/Jobs.tsx | 580 ++++++++++-------- apps/rlark-ui/src/pages/Login.tsx | 58 +- apps/rlark-ui/src/pages/Overview.tsx | 67 +- apps/rlark-ui/src/pages/SSHKeys.tsx | 68 +- apps/rlark-ui/src/pages/Storage.tsx | 580 ++++++++---------- apps/rlark-ui/src/pages/SystemConfig.tsx | 13 +- apps/rlark-ui/src/pages/Workflows.tsx | 53 +- apps/rlark-ui/src/styles/addons.css | 5 +- .../src/styles/clusters/management.css | 69 ++- .../src/styles/components/tag-and-filter.css | 64 +- apps/rlark-ui/src/styles/foundation/shell.css | 50 +- .../src/styles/jobs/code-editor-and-list.css | 48 +- apps/rlark-ui/src/styles/jobs/create.css | 19 + .../src/styles/jobs/detail-foundation.css | 43 +- .../src/styles/jobs/detail-runtime.css | 108 ++-- .../src/styles/jobs/tags-and-editing.css | 19 +- .../src/styles/nodes/files-and-selector.css | 4 + apps/rlark-ui/src/styles/nodes/insight.css | 11 +- .../styles/overrides/global-responsive.css | 2 +- apps/rlark-ui/src/styles/shared/content.css | 54 +- .../shared/table-actions-and-controls.css | 26 +- .../src/styles/storage/create-dialog.css | 126 +++- apps/rlark-ui/src/types.ts | 10 +- apps/rlark-ui/src/utils/clipboard.ts | 30 + apps/rlark-ui/src/utils/crd.ts | 174 +++--- apps/rlark-ui/src/utils/job.ts | 19 +- apps/rlark-ui/src/utils/jobPhase.ts | 13 + apps/rlark-ui/src/utils/nodeResources.ts | 9 +- apps/rlark-ui/src/utils/nodes.ts | 21 + apps/rlark-ui/src/utils/overlayContainer.ts | 15 + apps/rlark-ui/src/utils/route.ts | 8 +- apps/rlark-ui/src/utils/sshKeys.ts | 23 + apps/rlark-ui/src/utils/workerEvents.ts | 121 ++++ apps/rlark-ui/tests/api.test.mjs | 19 + apps/rlark-ui/tests/clipboard.test.mjs | 224 +++++++ .../tests/cluster-lifecycle-actions.test.mjs | 17 + .../tests/form-modal-dismiss.test.mjs | 55 ++ apps/rlark-ui/tests/image-reference.test.mjs | 5 +- apps/rlark-ui/tests/job-actions.test.mjs | 28 +- .../tests/job-clone-fidelity.test.mjs | 139 +++++ apps/rlark-ui/tests/job-header-role.test.mjs | 78 +++ apps/rlark-ui/tests/job-phase.test.mjs | 22 +- .../tests/job-role-count-and-copy.test.mjs | 107 ++++ apps/rlark-ui/tests/job-ssh-keys.test.mjs | 31 + apps/rlark-ui/tests/job-tag-filter.test.mjs | 106 ++++ apps/rlark-ui/tests/login-page.test.mjs | 40 +- apps/rlark-ui/tests/node-resources.test.mjs | 11 +- .../tests/node-worker-localization.test.mjs | 97 +++ apps/rlark-ui/tests/overlay-theme.test.mjs | 70 +++ apps/rlark-ui/tests/page-layout.test.mjs | 73 +++ .../tests/resource-placement.test.mjs | 78 ++- apps/rlark-ui/tests/route.test.mjs | 18 +- apps/rlark-ui/tests/worker-events.test.mjs | 203 ++++++ apps/rlark-ui/tsconfig.test.json | 4 +- apps/rlark/docs/admin-guide/agent.md | 10 +- .../admin-guide/cluster-delisted-migration.md | 26 + apps/rlark/docs/api/reference.md | 8 +- apps/rlark/docs/api/swagger.yaml | 108 ++++ apps/rlark/docs/architecture.md | 2 +- apps/rlark/docs/concepts.md | 6 +- .../docs/developer-guide/api-conventions.md | 2 +- .../docs/developer-guide/runtime-provider.md | 17 +- apps/rlark/docs/examples/update-kcp-crds.sh | 88 +++ apps/rlark/docs/reference/configuration.md | 1 - apps/rlark/docs/reference/crd.md | 4 +- apps/rlark/docs/storage-api.md | 59 +- apps/rlark/docs/user-guide/clusters-nodes.md | 15 +- apps/rlark/docs/user-guide/console.md | 32 +- apps/rlark/docs/user-guide/jobs.md | 27 +- apps/rlark/docs/user-guide/ssh-keys.md | 8 +- apps/rlark/docs/user-guide/storage.md | 4 +- apps/rlark/docs/zh/admin-guide/agent.md | 10 +- .../admin-guide/cluster-delisted-migration.md | 26 + apps/rlark/docs/zh/api/reference.md | 8 +- apps/rlark/docs/zh/architecture.md | 2 +- apps/rlark/docs/zh/concepts.md | 6 +- .../zh/developer-guide/api-conventions.md | 2 +- .../zh/developer-guide/runtime-provider.md | 15 +- apps/rlark/docs/zh/reference/configuration.md | 1 - apps/rlark/docs/zh/reference/crd.md | 2 +- apps/rlark/docs/zh/storage-api.md | 57 +- .../docs/zh/user-guide/clusters-nodes.md | 17 +- apps/rlark/docs/zh/user-guide/console.md | 32 +- apps/rlark/docs/zh/user-guide/jobs.md | 26 +- apps/rlark/docs/zh/user-guide/ssh-keys.md | 8 +- apps/rlark/docs/zh/user-guide/storage.md | 4 +- apps/rlark/pkg/agent/cluster_agent.go | 1 - apps/rlark/pkg/agent/config.go | 4 - apps/rlark/pkg/agent/config_test.go | 13 +- .../pkg/agent/controllers/task/api/types.go | 7 +- .../task/compatibility/kubernetes/task.go | 7 + .../compatibility/kubernetes/task_test.go | 1 - .../pkg/agent/controllers/task/controller.go | 4 - .../task/environment/kubernetes/ssh/ssh.go | 8 +- .../agent/controllers/task/execution_test.go | 34 +- .../agent/controllers/task/push_daemonset.go | 61 -- .../agent/controllers/task/push_deployment.go | 2 - .../task/workload/ray/kubernetes/adapter.go | 17 - .../workload/ray/kubernetes/adapter_test.go | 36 +- .../workload/ray/kubernetes/compatibility.go | 17 - .../task/workload/ray/kubernetes/observe.go | 8 - .../workload/ray/kubernetes/operations.go | 3 +- .../ray/kubernetes/operations_test.go | 3 - .../cluster/cluster_cleanup_test.go | 35 ++ .../cluster/cluster_controller.go | 366 ++++++++++- .../cluster/cluster_controller_test.go | 403 +++++++++++- .../cluster/cluster_drain_test.go | 248 ++++++++ apps/rlark/pkg/controllermanager/job/build.go | 43 +- .../job/job_controller_test.go | 74 +++ .../controllermanager/job/lifecycle_test.go | 95 ++- apps/rlark/pkg/controllermanager/job/sync.go | 8 +- .../pkg/controllermanager/job/sync_test.go | 17 + apps/rlark/pkg/gateway/api_reference.go | 36 ++ apps/rlark/pkg/gateway/auth_handler_test.go | 4 + apps/rlark/pkg/gateway/clusters_handler.go | 272 +++++++- .../pkg/gateway/clusters_handler_test.go | 322 ++++++++++ apps/rlark/pkg/gateway/kube_handler.go | 40 +- apps/rlark/pkg/gateway/kube_handler_test.go | 198 ++++++ apps/rlark/pkg/gateway/router.go | 24 +- apps/rlark/pkg/gateway/storage/bucket.go | 14 +- apps/rlark/pkg/gateway/storage/bucket_test.go | 48 ++ apps/rlark/pkg/gateway/storage/client.go | 29 +- apps/rlark/pkg/gateway/storage/provider.go | 138 +++++ .../pkg/gateway/storage/provider_test.go | 114 ++++ apps/rlark/pkg/gateway/storage_handler.go | 246 +++++--- .../rlark/pkg/gateway/storage_handler_test.go | 367 +++++++++++ apps/rlark/pkg/gateway/swagger_test.go | 8 +- .../pkg/rlarktools/raylauncher/launcher.go | 3 +- test/kindtest/.gitignore | 3 + test/kindtest/README.md | 357 +++++++++++ test/kindtest/config.example.yaml | 64 ++ test/kindtest/kindtest/__init__.py | 88 +++ test/kindtest/kindtest/__main__.py | 10 + test/kindtest/kindtest/cli.py | 165 +++++ test/kindtest/kindtest/config.py | 262 ++++++++ test/kindtest/kindtest/context.py | 562 +++++++++++++++++ test/kindtest/kindtest/kube.py | 193 ++++++ test/kindtest/kindtest/model.py | 144 +++++ test/kindtest/kindtest/registry.py | 280 +++++++++ test/kindtest/kindtest/report.py | 234 +++++++ test/kindtest/kindtest/runner.py | 447 ++++++++++++++ test/kindtest/kindtest/shell.py | 126 ++++ test/kindtest/kindtest/versions.py | 435 +++++++++++++ test/kindtest/kt | 14 + test/kindtest/test_framework.py | 404 ++++++++++++ test/kindtest/tests/00_env.py | 115 ++++ test/kindtest/tests/05_mig.py | 58 ++ test/kindtest/tests/10_ray.py | 143 +++++ test/kindtest/tests/20_task.py | 103 ++++ test/kindtest/tests/90_cleanup.py | 63 ++ 178 files changed, 12141 insertions(+), 2027 deletions(-) create mode 100644 apps/rlark-ui/src/utils/clipboard.ts create mode 100644 apps/rlark-ui/src/utils/overlayContainer.ts create mode 100644 apps/rlark-ui/src/utils/workerEvents.ts create mode 100644 apps/rlark-ui/tests/clipboard.test.mjs create mode 100644 apps/rlark-ui/tests/cluster-lifecycle-actions.test.mjs create mode 100644 apps/rlark-ui/tests/form-modal-dismiss.test.mjs create mode 100644 apps/rlark-ui/tests/job-clone-fidelity.test.mjs create mode 100644 apps/rlark-ui/tests/job-header-role.test.mjs create mode 100644 apps/rlark-ui/tests/job-role-count-and-copy.test.mjs create mode 100644 apps/rlark-ui/tests/job-tag-filter.test.mjs create mode 100644 apps/rlark-ui/tests/node-worker-localization.test.mjs create mode 100644 apps/rlark-ui/tests/overlay-theme.test.mjs create mode 100644 apps/rlark-ui/tests/worker-events.test.mjs create mode 100644 apps/rlark/docs/admin-guide/cluster-delisted-migration.md create mode 100755 apps/rlark/docs/examples/update-kcp-crds.sh create mode 100644 apps/rlark/docs/zh/admin-guide/cluster-delisted-migration.md delete mode 100644 apps/rlark/pkg/agent/controllers/task/push_daemonset.go create mode 100644 apps/rlark/pkg/controllermanager/cluster/cluster_cleanup_test.go create mode 100644 apps/rlark/pkg/controllermanager/cluster/cluster_drain_test.go create mode 100644 apps/rlark/pkg/gateway/storage/bucket_test.go create mode 100644 apps/rlark/pkg/gateway/storage/provider.go create mode 100644 apps/rlark/pkg/gateway/storage/provider_test.go create mode 100644 test/kindtest/.gitignore create mode 100644 test/kindtest/README.md create mode 100644 test/kindtest/config.example.yaml create mode 100644 test/kindtest/kindtest/__init__.py create mode 100644 test/kindtest/kindtest/__main__.py create mode 100644 test/kindtest/kindtest/cli.py create mode 100644 test/kindtest/kindtest/config.py create mode 100644 test/kindtest/kindtest/context.py create mode 100644 test/kindtest/kindtest/kube.py create mode 100644 test/kindtest/kindtest/model.py create mode 100644 test/kindtest/kindtest/registry.py create mode 100644 test/kindtest/kindtest/report.py create mode 100644 test/kindtest/kindtest/runner.py create mode 100644 test/kindtest/kindtest/shell.py create mode 100644 test/kindtest/kindtest/versions.py create mode 100755 test/kindtest/kt create mode 100644 test/kindtest/test_framework.py create mode 100644 test/kindtest/tests/00_env.py create mode 100644 test/kindtest/tests/05_mig.py create mode 100644 test/kindtest/tests/10_ray.py create mode 100644 test/kindtest/tests/20_task.py create mode 100644 test/kindtest/tests/90_cleanup.py diff --git a/api/config/crd/bases/rlinf.io_clusters.yaml b/api/config/crd/bases/rlinf.io_clusters.yaml index 9b3bbcc..6b23b70 100644 --- a/api/config/crd/bases/rlinf.io_clusters.yaml +++ b/api/config/crd/bases/rlinf.io_clusters.yaml @@ -57,9 +57,14 @@ spec: agentNamespace: description: AgentNamespace is the management-plane namespace that contains the agent heartbeat Lease and Node CRs. + minLength: 1 type: string cordoned: type: boolean + delisted: + type: boolean + drainPreviousCordoned: + type: boolean draining: type: boolean required: @@ -145,6 +150,7 @@ spec: - Downgraded - Cordoned - Draining + - Delisted type: string totalNodes: format: int32 diff --git a/api/kubeclients/applyconfiguration/rlark.io/v1alpha1/clusterspec.go b/api/kubeclients/applyconfiguration/rlark.io/v1alpha1/clusterspec.go index 428ce1e..b598a01 100644 --- a/api/kubeclients/applyconfiguration/rlark.io/v1alpha1/clusterspec.go +++ b/api/kubeclients/applyconfiguration/rlark.io/v1alpha1/clusterspec.go @@ -23,9 +23,11 @@ package v1alpha1 // ClusterSpec declares the durable operational state of a data-plane cluster. type ClusterSpecApplyConfiguration struct { // AgentNamespace is the management-plane namespace that contains the agent heartbeat Lease and Node CRs. - AgentNamespace *string `json:"agentNamespace,omitempty"` - Cordoned *bool `json:"cordoned,omitempty"` - Draining *bool `json:"draining,omitempty"` + AgentNamespace *string `json:"agentNamespace,omitempty"` + Cordoned *bool `json:"cordoned,omitempty"` + Draining *bool `json:"draining,omitempty"` + Delisted *bool `json:"delisted,omitempty"` + DrainPreviousCordoned *bool `json:"drainPreviousCordoned,omitempty"` } // ClusterSpecApplyConfiguration constructs a declarative configuration of the ClusterSpec type for use with @@ -57,3 +59,19 @@ func (b *ClusterSpecApplyConfiguration) WithDraining(value bool) *ClusterSpecApp b.Draining = &value return b } + +// WithDelisted sets the Delisted field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Delisted field is set to the value of the last call. +func (b *ClusterSpecApplyConfiguration) WithDelisted(value bool) *ClusterSpecApplyConfiguration { + b.Delisted = &value + return b +} + +// WithDrainPreviousCordoned sets the DrainPreviousCordoned field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DrainPreviousCordoned field is set to the value of the last call. +func (b *ClusterSpecApplyConfiguration) WithDrainPreviousCordoned(value bool) *ClusterSpecApplyConfiguration { + b.DrainPreviousCordoned = &value + return b +} diff --git a/api/rlark.io/v1alpha1/annotations.go b/api/rlark.io/v1alpha1/annotations.go index 621f4f7..f5da2b7 100644 --- a/api/rlark.io/v1alpha1/annotations.go +++ b/api/rlark.io/v1alpha1/annotations.go @@ -4,6 +4,11 @@ const ( WorkflowLabel = "rlinf.io/workflow" WorkflowStoppedByAnnotation = "rlinf.io/workflow-stopped-by" + ClusterStoppedAnnotation = "rlark.io/stopped" + // ClusterDrainSuspendedAnnotation records that Task.Spec.Suspended was set by a + // cluster drain, so cancelling the drain only resumes Tasks the drain suspended. + ClusterDrainSuspendedAnnotation = "rlark.io/drain-suspended" + RayRoleAnnotation = "rlark.io/ray-role" RayHeadTaskNameAnnotation = "rlark.io/ray-head-task-name" RayTotalNodesAnnotation = "rlark.io/ray-total-nodes" diff --git a/api/rlark.io/v1alpha1/cluster_types.go b/api/rlark.io/v1alpha1/cluster_types.go index 37f918b..c1bcbb6 100644 --- a/api/rlark.io/v1alpha1/cluster_types.go +++ b/api/rlark.io/v1alpha1/cluster_types.go @@ -5,7 +5,7 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // ClusterPhase summarizes the current lifecycle state of a data-plane cluster. // The set is closed in the CRD schema: adding a value requires updating the // enum marker, regenerating the CRD manifests, and updating the documentation. -// +kubebuilder:validation:Enum=Online;Offline;Downgraded;Cordoned;Draining +// +kubebuilder:validation:Enum=Online;Offline;Downgraded;Cordoned;Draining;Delisted type ClusterPhase string const ( @@ -14,14 +14,18 @@ const ( ClusterDowngraded ClusterPhase = "Downgraded" ClusterCordoned ClusterPhase = "Cordoned" ClusterDraining ClusterPhase = "Draining" + ClusterDelisted ClusterPhase = "Delisted" ) // ClusterSpec declares the durable operational state of a data-plane cluster. type ClusterSpec struct { // AgentNamespace is the management-plane namespace that contains the agent heartbeat Lease and Node CRs. - AgentNamespace string `json:"agentNamespace"` - Cordoned bool `json:"cordoned,omitempty"` - Draining bool `json:"draining,omitempty"` + // +kubebuilder:validation:MinLength=1 + AgentNamespace string `json:"agentNamespace"` + Cordoned bool `json:"cordoned,omitempty"` + Draining bool `json:"draining,omitempty"` + Delisted bool `json:"delisted,omitempty"` + DrainPreviousCordoned bool `json:"drainPreviousCordoned,omitempty"` } // ClusterStatus reports the controller's current observation of a data-plane cluster. diff --git a/apps/rlark-ui/src/App.tsx b/apps/rlark-ui/src/App.tsx index 44373f2..6b507d5 100644 --- a/apps/rlark-ui/src/App.tsx +++ b/apps/rlark-ui/src/App.tsx @@ -4,12 +4,7 @@ import type { Job } from "./data"; import { copy, type Lang, type Theme } from "./i18n"; import { navItems } from "./constants"; import type { Page } from "./types"; -import { - filesPath, - hasTerminalSession, - parseRoute, - useIsAdminPath, -} from "./utils/route"; +import { hasTerminalSession, parseRoute, useIsAdminPath } from "./utils/route"; import { Logo, Header, PlatformFooter } from "./components/shared"; import { TerminalPage } from "./components/terminal"; import { Overview } from "./pages/Overview"; @@ -75,13 +70,17 @@ export default function App() { document.title = isTerminal ? "WebTerminal · RLark" : isAdmin - ? "RLark 管理后台" - : "RLark具身智能云原生纳管平台"; + ? lang === "zh" + ? "RLark 管理后台" + : "RLark Admin Console" + : lang === "zh" + ? "RLark具身智能云原生纳管平台" + : "RLark Cloud-Native Embodied Intelligence Platform"; const favicon = document.querySelector("#app-favicon"); if (favicon) { favicon.href = "/favicon.png"; } - }, [isAdmin, isTerminal]); + }, [isAdmin, isTerminal, lang]); const navigate = ( next: Page, @@ -135,6 +134,7 @@ export default function App() { if (!userLoggedIn && !(isTerminal && terminalLoggedIn)) { return ( { setUserName(name); setUserLoggedIn(true); @@ -151,7 +151,9 @@ export default function App() { if (!workerCRName) { return (
- 缺少 Worker 参数，无法连接终端。 + {lang === "zh" + ? "缺少 Worker 参数，无法连接终端。" + : "Missing worker parameter, cannot open the terminal."}
); } @@ -161,6 +163,7 @@ export default function App() { workerName={workerCRName} jobName={jobName} workerStatus={workerStatus} + lang={lang} /> ); } @@ -179,7 +182,9 @@ export default function App() { const Icon = item.icon; const isParent = item.children && item.children.length > 0; const itemLabel = isParent ? c.nav.clustersParent : c.nav[item.id]; - const isActive = page === item.id; + const isActive = + page === item.id || + (item.id === "storageClass" && page === "files"); const isChildActive = isParent ? item.children!.some((ch) => ch.id === page) : false; @@ -353,9 +358,7 @@ export default function App() { navigate("storageClass", name, { replace: !name }) } onCreate={() => setStorageCreateOpen(true)} - onBrowseFiles={(cluster, storageClass) => - window.open(filesPath(cluster, storageClass), "_blank") - } + onBrowseFiles={(storageClass) => navigate("files", storageClass)} refreshKey={storageRefreshKey} /> )} diff --git a/apps/rlark-ui/src/admin/Addons.tsx b/apps/rlark-ui/src/admin/Addons.tsx index 8c819e3..471b021 100644 --- a/apps/rlark-ui/src/admin/Addons.tsx +++ b/apps/rlark-ui/src/admin/Addons.tsx @@ -59,6 +59,26 @@ export function AddonsPage({ lang }: { lang: Lang }) { } }; + // Addon.status.phase values reported by the addon controller. Unknown values + // are rendered verbatim so new phases still surface in the UI. + const phaseLabel = (phase: string) => { + if (!zh) return phase; + switch (phase) { + case "Ready": + return "就绪"; + case "Failed": + return "失败"; + case "Installing": + return "安装中"; + case "Upgrading": + return "升级中"; + case "Pending": + return "等待中"; + default: + return phase; + } + }; + const totalPages = Math.ceil(installed.length / pageSize); const pagedInstalled = installed.slice( (page - 1) * pageSize, @@ -209,7 +229,7 @@ export function AddonsPage({ lang }: { lang: Lang }) { {a.status?.version || a.spec?.version || "-"} - {a.status?.phase || "Pending"} + {phaseLabel(a.status?.phase || "Pending")} {a.status?.message || "-"} diff --git a/apps/rlark-ui/src/admin/AdminApp.tsx b/apps/rlark-ui/src/admin/AdminApp.tsx index 269c53e..8ec1bb0 100644 --- a/apps/rlark-ui/src/admin/AdminApp.tsx +++ b/apps/rlark-ui/src/admin/AdminApp.tsx @@ -27,7 +27,7 @@ import { SSHKeysPage } from "../pages/SSHKeys"; import { ImageRegistriesPage } from "../pages/ImageRegistries"; import { SystemConfigPage } from "../pages/SystemConfig"; import { JobsPage } from "../pages/Jobs"; -import { filesPath, parseAdminRoute } from "../utils/route"; +import { parseAdminRoute } from "../utils/route"; import { clearAuthSession, hasAuthSession, @@ -285,7 +285,9 @@ export function AdminApp() { const Icon = item.icon; const itemLabel = zh ? item.zh : item.en; const isParent = item.children && item.children.length > 0; - const isActive = adminPage === item.id; + const isActive = + adminPage === item.id || + (item.id === "storageClass" && adminPage === "files"); const isChildActive = isParent ? item.children!.some((ch) => ch.id === adminPage) : false; @@ -396,6 +398,7 @@ export function AdminApp() { selectedClusterID={adminSub} onSelectCluster={(id?: string) => navigate("clusters-list", id)} onSelectNode={(name: string) => navigate("clusters-nodes", name)} + adminMode /> )} {adminPage === "jobs" && ( @@ -431,9 +434,7 @@ export function AdminApp() { selectedName={adminSub} onSelect={(name?: string) => navigate("storageClass", name)} onCreate={() => navigate("storageClass", "create")} - onBrowseFiles={(cluster, storageClass) => - window.open(filesPath(cluster, storageClass, true), "_blank") - } + onBrowseFiles={(storageClass) => navigate("files", storageClass)} refreshKey={storageRefreshKey} /> )} diff --git a/apps/rlark-ui/src/admin/AdminPage.tsx b/apps/rlark-ui/src/admin/AdminPage.tsx index ebe1b06..ad23252 100644 --- a/apps/rlark-ui/src/admin/AdminPage.tsx +++ b/apps/rlark-ui/src/admin/AdminPage.tsx @@ -239,7 +239,7 @@ export function ClustersOverviewAdminPage({ copy: c }: { copy: Copy }) {
- Distribution + {zh ? "分布" : "Distribution"}

{zh ? "集群分布" : "Cluster Distribution"}

@@ -1595,7 +1595,7 @@ export function AdminPage({ (category) => categoryLabels[category][zh ? "zh" : "en"], ) - .join("、") + .join(zh ? "、" : ", ") : zh ? "当前未设置" : "Not set"} diff --git a/apps/rlark-ui/src/admin/CreateCluster.tsx b/apps/rlark-ui/src/admin/CreateCluster.tsx index c62f338..177b16a 100644 --- a/apps/rlark-ui/src/admin/CreateCluster.tsx +++ b/apps/rlark-ui/src/admin/CreateCluster.tsx @@ -15,6 +15,7 @@ import { systemConfigApi, type DeploymentConfig, } from "../backend"; +import { copyText } from "../utils/clipboard"; import { buildDeployYaml } from "../utils/deployYaml"; export function CreateClusterPage({ lang }: { lang: Lang }) { @@ -72,11 +73,10 @@ export function CreateClusterPage({ lang }: { lang: Lang }) { const deployYaml = result ? buildDeployYaml(result, deploymentConfig) : ""; - const handleCopy = () => { - navigator.clipboard.writeText(deployYaml).then(() => { - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }); + const handleCopy = async () => { + if (!(await copyText(deployYaml))) return; + setCopied(true); + setTimeout(() => setCopied(false), 2000); }; const handleExpand = async (cid: string) => { @@ -92,14 +92,12 @@ export function CreateClusterPage({ lang }: { lang: Lang }) { } catch {} }; - const handleExpandedCopy = () => { + const handleExpandedCopy = async () => { if (!expandedResult) return; - navigator.clipboard - .writeText(buildDeployYaml(expandedResult, deploymentConfig)) - .then(() => { - setExpandedCopied(true); - setTimeout(() => setExpandedCopied(false), 2000); - }); + const yaml = buildDeployYaml(expandedResult, deploymentConfig); + if (!(await copyText(yaml))) return; + setExpandedCopied(true); + setTimeout(() => setExpandedCopied(false), 2000); }; return ( diff --git a/apps/rlark-ui/src/backend.ts b/apps/rlark-ui/src/backend.ts index f917238..a1a3f58 100644 --- a/apps/rlark-ui/src/backend.ts +++ b/apps/rlark-ui/src/backend.ts @@ -120,6 +120,21 @@ export const clustersApi = { ); return response.data; }, + lifecycle( + id: string, + action: "cordon" | "uncordon" | "drain" | "undrain" | "delist" | "restore", + ) { + return requestJson>( + `/api/v1/clusters/${encodeURIComponent(id)}/${action}`, + { method: "POST" }, + ); + }, + remove(id: string) { + return requestJson>( + `/api/v1/clusters/${encodeURIComponent(id)}/delete`, + { method: "POST" }, + ); + }, }; function resourceApi(resource: string) { @@ -302,6 +317,14 @@ export const storageClassesApi = { ); return response.data ?? {}; }, + // providers 返回后端维护的提供商目录；value 是提交给后端的合法取值， + // name 仅用于展示。前端不再硬编码该列表。 + async providers() { + const response = await requestJson< + DataResponse> + >("/api/v1/storage/storageclass/provider"); + return response.data ?? []; + }, create(body: object) { return request("/api/v1/storage/storageclass", { method: "POST", body }); }, @@ -318,35 +341,35 @@ export const storageClassesApi = { }, }; -const storageObjectPath = (storageClass: string, cluster: string) => - `/api/v1/storage/storageclass/${encodeURIComponent(storageClass)}/${encodeURIComponent(cluster)}`; +const storageObjectPath = (storageClass: string) => + `/api/v1/storage/storageclass/${encodeURIComponent(storageClass)}`; export const storageObjectsApi = { - list(storageClass: string, cluster: string, prefix: string) { + list(storageClass: string, prefix: string) { return requestJson( - withQuery(`${storageObjectPath(storageClass, cluster)}/list`, { + withQuery(`${storageObjectPath(storageClass)}/list`, { prefix, maxKeys: 100, }), ); }, - upload(storageClass: string, cluster: string, formData: FormData) { - return request(`${storageObjectPath(storageClass, cluster)}/upload`, { + upload(storageClass: string, formData: FormData) { + return request(`${storageObjectPath(storageClass)}/upload`, { method: "POST", body: formData, }); }, - download(storageClass: string, cluster: string, key: string) { + download(storageClass: string, key: string) { return requestJson( withQuery( - `${storageObjectPath(storageClass, cluster)}/object/${encodeURIComponent(key)}`, + `${storageObjectPath(storageClass)}/object/${encodeURIComponent(key)}`, { expire: 3600 }, ), ); }, - remove(storageClass: string, cluster: string, key: string) { + remove(storageClass: string, key: string) { return request( - `${storageObjectPath(storageClass, cluster)}/object/${encodeURIComponent(key)}`, + `${storageObjectPath(storageClass)}/object/${encodeURIComponent(key)}`, { method: "DELETE" }, ); }, diff --git a/apps/rlark-ui/src/components/NodeResourceBrowser.tsx b/apps/rlark-ui/src/components/NodeResourceBrowser.tsx index 88b38af..41359d3 100644 --- a/apps/rlark-ui/src/components/NodeResourceBrowser.tsx +++ b/apps/rlark-ui/src/components/NodeResourceBrowser.tsx @@ -493,7 +493,7 @@ export function NodeResourceBrowser({ category !== "unknown", ); categories.forEach((category) => totalByCat[category]++); + const city = getNodeLocation(node); + if (!city) continue; + const ns = node.metadata.namespace ?? ""; if (!cityMap.has(city)) { cityMap.set(city, { diff --git a/apps/rlark-ui/src/components/ResourcePlacementPicker.tsx b/apps/rlark-ui/src/components/ResourcePlacementPicker.tsx index c0996ca..5742916 100644 --- a/apps/rlark-ui/src/components/ResourcePlacementPicker.tsx +++ b/apps/rlark-ui/src/components/ResourcePlacementPicker.tsx @@ -24,6 +24,11 @@ import { type Mode = "model" | "manual"; type ResourceKind = "gpu" | "device" | "compute"; +// 型号缺失时用占位标识参与分组，展示时再按语言翻译，避免把中文写进数据层。 +const UNLABELED_GPU = "__unlabeled_gpu__"; +const unlabeledGPULabel = (zh: boolean) => + zh ? "未标注 GPU" : "Unlabeled GPU"; + type NodeResource = { node: CRDNodeLite; kind: ResourceKind; @@ -76,7 +81,7 @@ function nodeResources( resources.push({ node, kind: "gpu", - model: getNodeGPUModel(fullNode) || "未标注 GPU", + model: getNodeGPUModel(fullNode) || UNLABELED_GPU, resourceKey: gpuKey, total: gpuTotal, free: availableResource( @@ -123,6 +128,44 @@ function nodeResources( return resources; } +// 整数输入框：编辑期间保留用户的原始输入（包括空字符串），失焦后再归一化 +// 回受控数值。直接用 value={number} + Number(event.target.value) 的写法会在 +// 用户按删除键清空最后一位时立刻把 "" 解析成 min 并写回，导致输入框永远无法 +// 清空（只能变成 0/1），并且在 "0" 这类前缀输入时让 DOM 与 state 失去同步 +// （输入框残留 "01"）。 +function IntegerField({ + value, + min, + disabled, + onCommit, +}: { + value: number; + min: number; + disabled?: boolean; + onCommit: (next: number) => void; +}) { + const [draft, setDraft] = useState(null); + return ( + { + const raw = event.target.value; + setDraft(raw); + const parsed = Number(raw); + onCommit( + raw.trim() === "" || !Number.isFinite(parsed) + ? min + : Math.max(min, Math.trunc(parsed)), + ); + }} + onBlur={() => setDraft(null)} + /> + ); +} + export function ResourcePlacementPicker({ zh, cluster, @@ -310,15 +353,17 @@ export function ResourcePlacementPicker({ const kind = selectedOption?.kind ?? "compute"; const model = selectedOption?.model ?? ""; const displayModel = - model === "__unset_device_model__" - ? zh - ? "未设置设备型号" - : "Device model not set" - : model === "__undiscovered_device__" + model === UNLABELED_GPU + ? unlabeledGPULabel(zh) + : model === "__unset_device_model__" ? zh - ? "未发现设备" - : "No device discovered" - : model; + ? "未设置设备型号" + : "Device model not set" + : model === "__undiscovered_device__" + ? zh + ? "未发现设备" + : "No device discovered" + : model; const isUnconfiguredDevice = kind === "device" && selectedOption?.resources.every((resource) => !resource.configured); @@ -417,7 +462,7 @@ export function ResourcePlacementPicker({ setSelected(new Set()); }; - const commitSelection = (next: Set) => { + const commitSelection = (next: Set, amount = resourceAmount) => { setSelected(next); const selectedResources = candidates.filter((item) => next.has(item.node.metadata.name), @@ -431,14 +476,27 @@ export function ResourcePlacementPicker({ }) : "", replicas: next.size, - gpu: kind === "gpu" ? String(resourceAmount) : "0", + gpu: kind === "gpu" ? String(amount) : "0", devices: kind === "device" && resourceKey - ? [{ name: resourceKey, quantity: String(resourceAmount) }] + ? [{ name: resourceKey, quantity: String(amount) }] : [], }); }; + // 改规格不是重新选节点：只剔除空闲量不再满足新规格的节点，其余选择保留。 + // 输入过程中的空值会被归一化成 min，因此中间态不会误伤已选节点。 + const selectionWithin = (amount: number) => + new Set( + candidates + .filter( + (resource) => + selected.has(resource.node.metadata.name) && + (!resource.configured || resource.free >= amount), + ) + .map((resource) => resource.node.metadata.name), + ); + const handleSelectionStart = (event: ReactPointerEvent) => { if (event.button !== 0 || !gridRef.current) return; event.preventDefault(); @@ -588,15 +646,17 @@ export function ResourcePlacementPicker({ : "General compute"} - {option.model === "__undiscovered_device__" - ? zh - ? "未发现设备" - : "No device discovered" - : option.model === "__unset_device_model__" + {option.model === UNLABELED_GPU + ? unlabeledGPULabel(zh) + : option.model === "__undiscovered_device__" ? zh - ? "未设置设备型号" - : "Device model not set" - : option.model} + ? "未发现设备" + : "No device discovered" + : option.model === "__unset_device_model__" + ? zh + ? "未设置设备型号" + : "Device model not set" + : option.model} @@ -620,32 +680,16 @@ export function ResourcePlacementPicker({ ? `单 Worker ${kind === "gpu" ? "GPU" : "设备"}数量` : "Resources per worker"} - { - const nextAmount = Math.max(0, Number(event.target.value)); + onCommit={(nextAmount) => { setResourceAmount(nextAmount); if (mode === "model") { commitModel(Math.max(1, replicas), nextAmount); } else { - setSelected(new Set()); - onChange({ - nodeSelector: "", - replicas: 0, - gpu: kind === "gpu" ? String(nextAmount) : "0", - devices: - kind === "device" && candidates[0]?.resourceKey - ? [ - { - name: candidates[0].resourceKey, - quantity: String(nextAmount), - }, - ] - : [], - }); + commitSelection(selectionWithin(nextAmount), nextAmount); } }} /> @@ -720,15 +764,11 @@ export function ResourcePlacementPicker({ {zh ? "Worker 数量" : "Workers"} - - commitModel( - Math.max(1, Number(event.target.value)), - resourceAmount, - ) + min={1} + onCommit={(nextReplicas) => + commitModel(nextReplicas, resourceAmount) } /> @@ -833,15 +873,17 @@ export function ResourcePlacementPicker({
- {resource.model === "__unset_device_model__" - ? zh - ? "未设置设备型号" - : "Device model not set" - : resource.configured - ? resource.model - : zh - ? "未分类 · 未发现设备" - : "Unclassified · No device discovered"} + {resource.model === UNLABELED_GPU + ? unlabeledGPULabel(zh) + : resource.model === "__unset_device_model__" + ? zh + ? "未设置设备型号" + : "Device model not set" + : resource.configured + ? resource.model + : zh + ? "未分类 · 未发现设备" + : "Unclassified · No device discovered"} {resource.free} / {resource.total} @@ -894,7 +936,7 @@ export function ResourcePlacementPicker({ ? zh ? `通用计算 · ${model}` : `General compute · ${model}` - : `${zh ? "单 Worker" : "Per worker"}：${resourceAmount} × ${displayModel}`} + : `${zh ? "单 Worker：" : "Per worker: "}${resourceAmount} × ${displayModel}`}
{mode === "model" ? ( diff --git a/apps/rlark-ui/src/components/TagEditor.tsx b/apps/rlark-ui/src/components/TagEditor.tsx index cb5ad74..9e82ba4 100644 --- a/apps/rlark-ui/src/components/TagEditor.tsx +++ b/apps/rlark-ui/src/components/TagEditor.tsx @@ -8,6 +8,7 @@ import { import { createPortal } from "react-dom"; import { ChevronDown, Plus, Trash2, X } from "lucide-react"; import type { JobTag } from "../data"; +import { overlayContainer } from "../utils/overlayContainer"; export interface TagSuggestion { key: string; @@ -632,7 +633,7 @@ function Combobox({ ))} , - document.body, + overlayContainer(), )} ); diff --git a/apps/rlark-ui/src/components/TagFilterPopover.tsx b/apps/rlark-ui/src/components/TagFilterPopover.tsx index 4e283ae..884db1c 100644 --- a/apps/rlark-ui/src/components/TagFilterPopover.tsx +++ b/apps/rlark-ui/src/components/TagFilterPopover.tsx @@ -28,7 +28,9 @@ export function TagFilterPopover({ onClose, }: TagFilterPopoverProps) { const [search, setSearch] = useState(""); - const [hoveredKey, setHoveredKey] = useState(null); + // 右栏展示哪个 key 的 value 列表。只由点击切换：悬停切换太灵敏，指针从左栏 + // 移向右栏的途中会扫过其他 key，右栏内容跟着跳变，下方 key 的 value 就勾不中。 + const [activeKey, setActiveKey] = useState(null); const [valuePage, setValuePage] = useState(1); const popoverRef = useRef(null); @@ -78,8 +80,6 @@ export function TagFilterPopover({ return () => document.removeEventListener("mousedown", handler); }, [onClose]); - const allKeys = [...grouped.keys()]; - const keySelected = (k: string) => selection[k] !== undefined && selection[k].length > 0; @@ -114,27 +114,6 @@ export function TagFilterPopover({ onReset(); }; - const toggleSelectAllKeys = () => { - const newSel: Record = {}; - // 如果全部已选则清空，否则全选 - let allSelected = allKeys.length > 0; - for (const k of allKeys) { - const sel = selection[k]; - if (!sel || sel.length !== (grouped.get(k)?.length ?? 0)) { - allSelected = false; - break; - } - } - if (allSelected) { - onChange({}); - } else { - for (const k of allKeys) { - newSel[k] = grouped.get(k) ?? []; - } - onChange(newSel); - } - }; - // 计算定位位置 const style: React.CSSProperties = {}; if (anchorRect) { @@ -147,10 +126,14 @@ export function TagFilterPopover({ style.top = anchorRect.bottom + 6; } - const effectiveHover = hoveredKey ?? filteredKeys.find(keySelected) ?? null; + // activeKey 可能因搜索过滤而不在当前列表中，此时才回退到已选中的 key。 + const effectiveKey = + activeKey && filteredKeys.includes(activeKey) + ? activeKey + : (filteredKeys.find(keySelected) ?? null); const selectedValues = useMemo( - () => (effectiveHover ? (grouped.get(effectiveHover) ?? []) : []), - [effectiveHover, grouped], + () => (effectiveKey ? (grouped.get(effectiveKey) ?? []) : []), + [effectiveKey, grouped], ); // 已选中 key 时，搜索同时筛选该 key 下不符合的 value const matchedValues = useMemo(() => { @@ -158,8 +141,8 @@ export function TagFilterPopover({ if (!q) return selectedValues; return selectedValues.filter((v) => v.toLowerCase().includes(q)); }, [selectedValues, search]); - const selectedValsForHover = effectiveHover - ? (selection[effectiveHover] ?? []) + const selectedValsForKey = effectiveKey + ? (selection[effectiveKey] ?? []) : []; const valuePageSize = 10; const valuePageCount = Math.max( @@ -174,7 +157,7 @@ export function TagFilterPopover({ useEffect(() => { setValuePage(1); - }, [effectiveHover, search]); + }, [effectiveKey, search]); return (
setSearch(e.target.value)} />
+

+ {zh + ? "多个标签 key 之间为「且」，同一 key 下的多个值之间为「或」。" + : "Different keys are ANDed; values within the same key are ORed."} +

{/* 左栏：key 列表 */}
- + + {zh ? "标签 key" : "Keys"} +
{filteredKeys.length === 0 ? (
@@ -216,31 +200,33 @@ export function TagFilterPopover({ const totalVals = grouped.get(k)?.length ?? 0; const selVals = selection[k]?.length ?? 0; return ( - + +
); }) )} @@ -250,20 +236,29 @@ export function TagFilterPopover({
- {effectiveHover + {effectiveKey ? zh - ? `标签值 — ${effectiveHover}` - : `Values — ${effectiveHover}` + ? `标签值 — ${effectiveKey}` + : `Values — ${effectiveKey}` : zh ? "标签值" : "Values"} + {effectiveKey && selectedValues.length > 0 && ( + + )}
- {!effectiveHover || selectedValues.length === 0 ? ( + {!effectiveKey || selectedValues.length === 0 ? (
{zh - ? "选择左侧的 key 查看对应的值" - : "Select a key on the left to see its values"} + ? "点击左侧的 key 查看对应的值" + : "Click a key on the left to see its values"}
) : matchedValues.length === 0 ? (
@@ -275,8 +270,8 @@ export function TagFilterPopover({ diff --git a/apps/rlark-ui/src/components/terminal.tsx b/apps/rlark-ui/src/components/terminal.tsx index 08202d7..384512e 100644 --- a/apps/rlark-ui/src/components/terminal.tsx +++ b/apps/rlark-ui/src/components/terminal.tsx @@ -13,28 +13,38 @@ import { } from "../utils/terminalKeyboard"; import "@xterm/xterm/css/xterm.css"; import { terminalApi } from "../backend"; +import type { Lang } from "../i18n"; -const workerStatusLabels: Record = { - Running: "运行中", - Pending: "等待中", - Succeeded: "已完成", - Failed: "失败", - Stopped: "已停止", - Online: "在线", - Offline: "离线", +const workerStatusLabels: Record = { + Running: { zh: "运行中", en: "Running" }, + Pending: { zh: "等待中", en: "Pending" }, + Succeeded: { zh: "已完成", en: "Succeeded" }, + Failed: { zh: "失败", en: "Failed" }, + Stopped: { zh: "已停止", en: "Stopped" }, + Online: { zh: "在线", en: "Online" }, + Offline: { zh: "离线", en: "Offline" }, }; +function workerStatusLabel(status: string, zh: boolean): string { + const label = workerStatusLabels[status]; + if (!label) return status; + return zh ? label.zh : label.en; +} + export function TerminalPage({ workerCRName, workerName, jobName, workerStatus, + lang, }: { workerCRName: string; workerName: string; jobName: string; workerStatus: string; + lang: Lang; }) { + const zh = lang === "zh"; const termRef = useRef(null); const wsRef = useRef(null); const termRefInner = useRef(null); @@ -61,7 +71,9 @@ export function TerminalPage({ term.focus(); termRefInner.current = term; - term.writeln(`Connecting to ${workerName} ...`); + term.writeln( + zh ? `正在连接 ${workerName} ...` : `Connecting to ${workerName} ...`, + ); const ws = terminalApi.createSocket(workerCRName); wsRef.current = ws; @@ -87,7 +99,11 @@ export function TerminalPage({ ws.onopen = () => { setConnectionState("connected"); - term.writeln("\r\nConnected. Starting shell ...\r\n"); + term.writeln( + zh + ? "\r\n已连接，正在启动 Shell ...\r\n" + : "\r\nConnected. Starting shell ...\r\n", + ); sendResize(); }; ws.onmessage = (e) => { @@ -118,7 +134,9 @@ export function TerminalPage({ downloadChunks = []; downloadName = msg.name || "download"; setTransferStatus( - `Downloading ${downloadName} (${msg.size || 0} bytes)...`, + zh + ? `正在下载 ${downloadName}（${msg.size || 0} 字节）...` + : `Downloading ${downloadName} (${msg.size || 0} bytes)...`, ); return; } @@ -136,17 +154,28 @@ export function TerminalPage({ downloading = false; downloadChunks = []; setTransferStatus(""); - term.writeln(`\r\n\x1b[32mDownloaded ${downloadName}.\x1b[0m`); + term.writeln( + zh + ? `\r\n\x1b[32m已下载 ${downloadName}。\x1b[0m` + : `\r\n\x1b[32mDownloaded ${downloadName}.\x1b[0m`, + ); } return; } if (msg.type === "file-transfer-done") { setTransferStatus(""); if (msg.success) { - term.writeln("\r\n\x1b[32mFile transfer complete.\x1b[0m"); + term.writeln( + zh + ? "\r\n\x1b[32m文件传输完成。\x1b[0m" + : "\r\n\x1b[32mFile transfer complete.\x1b[0m", + ); } else { + const detail = msg.error || (zh ? "未知错误" : "unknown"); term.writeln( - `\r\n\x1b[31mFile transfer failed: ${msg.error || "unknown"}\x1b[0m`, + zh + ? `\r\n\x1b[31m文件传输失败：${detail}\x1b[0m` + : `\r\n\x1b[31mFile transfer failed: ${detail}\x1b[0m`, ); } return; @@ -180,14 +209,22 @@ export function TerminalPage({ terminalProcessExitCode ?? getTerminalCloseExitCode(event.reason); if (exitCode !== null) { term.writeln( - `\r\n\x1b[33mSession ended (exit code ${exitCode}).\x1b[0m`, + zh + ? `\r\n\x1b[33m会话已结束（退出码 ${exitCode}）。\x1b[0m` + : `\r\n\x1b[33mSession ended (exit code ${exitCode}).\x1b[0m`, ); } else if (event.code === 1000 || legacyProxyCloseReceived) { - term.writeln("\r\n\x1b[33mSession ended.\x1b[0m"); + term.writeln( + zh + ? "\r\n\x1b[33m会话已结束。\x1b[0m" + : "\r\n\x1b[33mSession ended.\x1b[0m", + ); } else { const reason = event.reason ? `: ${event.reason}` : ""; term.writeln( - `\r\n\x1b[31mConnection closed unexpectedly (${event.code})${reason}.\x1b[0m`, + zh + ? `\r\n\x1b[31m连接意外中断（${event.code}）${reason}。\x1b[0m` + : `\r\n\x1b[31mConnection closed unexpectedly (${event.code})${reason}.\x1b[0m`, ); } }; @@ -223,7 +260,7 @@ export function TerminalPage({ term.dispose(); termRefInner.current = null; }; - }, [workerCRName, workerName]); + }, [workerCRName, workerName, zh]); const handleUploadClick = () => { fileInputRef.current?.click(); @@ -236,7 +273,9 @@ export function TerminalPage({ const ws = wsRef.current; const destPath = `./${file.name}`; setTransferStatus( - `Uploading ${file.name} (${file.size} bytes) to ${destPath}...`, + zh + ? `正在上传 ${file.name}（${file.size} 字节）到 ${destPath}...` + : `Uploading ${file.name} (${file.size} bytes) to ${destPath}...`, ); ws.send( @@ -282,7 +321,7 @@ export function TerminalPage({ ) return; wsRef.current.send(JSON.stringify({ type: "file-download", path: dlPath })); - setTransferStatus(`Requesting ${dlPath}...`); + setTransferStatus(zh ? `正在请求 ${dlPath}...` : `Requesting ${dlPath}...`); setShowDownloadInput(false); setDlPath(""); }; @@ -303,34 +342,40 @@ export function TerminalPage({ - {workerStatusLabels[workerStatus] ?? workerStatus} + {workerStatusLabel(workerStatus, zh)} )} {connectionState === "connected" - ? "已连接" + ? zh + ? "已连接" + : "Connected" : connectionState === "connecting" - ? "连接中" - : "连接已断开"} + ? zh + ? "连接中" + : "Connecting" + : zh + ? "连接已断开" + : "Disconnected"}
@@ -341,7 +386,11 @@ export function TerminalPage({ value={dlPath} onChange={(e) => setDlPath(e.target.value)} onKeyDown={(e) => e.key === "Enter" && handleDownloadSubmit()} - placeholder="输入 Pod 内的文件路径，例如 /workspace/output.log" + placeholder={ + zh + ? "输入 Pod 内的文件路径，例如 /workspace/output.log" + : "Enter a file path inside the Pod, e.g. /workspace/output.log" + } autoFocus />
)} diff --git a/apps/rlark-ui/src/crdModels.ts b/apps/rlark-ui/src/crdModels.ts index d74fce6..7b76d40 100644 --- a/apps/rlark-ui/src/crdModels.ts +++ b/apps/rlark-ui/src/crdModels.ts @@ -9,3 +9,10 @@ export type CRDWorkflow = IWorkflow; export type CRDJobTask = NonNullable< NonNullable["tasks"] >[number]; + +/** A single entry of the Pod template `spec.volumes` list carried by a Job task. */ +export type CRDVolume = NonNullable< + NonNullable< + NonNullable["spec"] + >["volumes"] +>[number]; diff --git a/apps/rlark-ui/src/data.ts b/apps/rlark-ui/src/data.ts index 4ab1914..29eb291 100644 --- a/apps/rlark-ui/src/data.ts +++ b/apps/rlark-ui/src/data.ts @@ -1,5 +1,13 @@ import type { NodeEventEntry } from "./types"; -import type { CRDWorkflow } from "./crdModels"; +import type { CRDVolume, CRDWorkflow } from "./crdModels"; + +/** + * Mount kinds the create/clone form can express. + * `opaque` covers every other Kubernetes volume source (emptyDir, configMap, + * secret, projected, ...). Such volumes are preserved verbatim in `rawVolume` + * so a clone round-trip does not silently rewrite them. + */ +export type MountType = "host" | "storage" | "opaque"; export type Phase = | "Running" @@ -11,7 +19,8 @@ export type Phase = | "Deleting" | "Unknown" | "Online" - | "Offline"; + | "Offline" + | "Downgraded"; export type ClusterType = "Cloud" | "Embodied"; export type NodeKind = "CloudCompute" | "EmbodiedCompute" | "Robot"; @@ -86,6 +95,10 @@ export interface Worker { // Task.status.events 聚合，仅在 worker 处于 Pending 时填充，供状态 // 徽标 "i" tooltip 展示。 events?: NodeEventEntry[]; + // 已被后续进展取代的历史事件（如 Pod 绑定节点之前的 FailedScheduling）。 + // 与 events 分开存放，tooltip 以"已恢复"的弱化样式单独分区展示，既不把 + // 历史告警误报成当前原因，也不丢失排查线索。 + resolvedEvents?: NodeEventEntry[]; // Task 名称，用于获取节点 RANK taskName?: string; } @@ -120,10 +133,11 @@ export interface Job { command: string; env: Array<{ key: string; value: string }>; mounts: Array<{ - type: "host" | "storage"; + type: MountType; objectStorage: string; mountPath: string; hostPath: string; + rawVolume?: CRDVolume; }>; headerRole: string; headerWorker: string; @@ -146,11 +160,12 @@ export interface Job { prepareScript: string; env: Array<{ key: string; value: string }>; mounts: Array<{ - type: "host" | "storage"; + type: MountType; objectStorage: string; mountPath: string; hostPath: string; pvcSizeGb: number; + rawVolume?: CRDVolume; }>; }>; taskStatuses: Array<{ @@ -184,20 +199,19 @@ export interface Domain { createdAt: string; } -export type StorageProvider = - | "AWS S3" - | "Google Cloud" - | "Azure Blob" - | "MinIO" - | "Aliyun OSS" - | "Tencent COS"; +/** + * Provider 取值由后端 `/api/v1/storage/storageclass/provider` 维护， + * 前端不再枚举：旧版枚举里混入了展示名和非 S3 的后端，提交后会被下发成 + * rclone 无法识别的配置。 + */ +export type StorageProvider = string; export interface StorageClass { id: string; name: string; namespace: string; - provider: StorageProvider; clusters: string[]; + provider: StorageProvider; endpoint: string; region: string; bucket: string; @@ -211,7 +225,6 @@ export interface StorageClassCreateRequest { name: string; namespace: string; provider: StorageProvider; - clusters: string[]; endpoint: string; region: string; bucket: string; @@ -301,8 +314,8 @@ export const storageClasses: StorageClass[] = [ id: "training-datasets", name: "training-datasets", namespace: "default", + clusters: ["cloud-east-a"], provider: "MinIO", - clusters: clusters.map((cluster) => cluster.id), endpoint: "https://minio.mock.local", region: "local", bucket: "rlark-training", @@ -315,8 +328,8 @@ export const storageClasses: StorageClass[] = [ id: "evaluation-results", name: "evaluation-results", namespace: "default", + clusters: ["cloud-east-a"], provider: "AWS S3", - clusters: clusters.slice(0, 2).map((cluster) => cluster.id), endpoint: "https://s3.mock.local", region: "cn-east-1", bucket: "rlark-evaluation", diff --git a/apps/rlark-ui/src/generated/rlark/_schemas/RlinfIoV1alpha1Cluster.js b/apps/rlark-ui/src/generated/rlark/_schemas/RlinfIoV1alpha1Cluster.js index 3ceb2d2..f10281a 100644 --- a/apps/rlark-ui/src/generated/rlark/_schemas/RlinfIoV1alpha1Cluster.js +++ b/apps/rlark-ui/src/generated/rlark/_schemas/RlinfIoV1alpha1Cluster.js @@ -2,7 +2,7 @@ import { formats } from "@kubernetes-models/validate"; export const validate = validate2; -const schema2 = {"type":"object","properties":{"apiVersion":{"$ref":"rgSv5V2hoWAdXQ_XErlwxUiy6phhz4ZEBCjtqeXtVec"},"kind":{"$ref":"KpP0Pr-LdgzYZe_Ch7AbCL6-S9umLrC7dsOL9rmrynU"},"metadata":{"$ref":"P3srTZpWadQH4s4N9KfJR0iCqydQo9YAz7xMB2qf0GQ"},"spec":{"$ref":"K6ju3c95K9dakJW2FdeROYd6DHeuFSxjmudg4NlnB24"},"status":{"$ref":"9AefQ5_3z-HVNG7_9qubG5OYd4JV9ucQKUu7a9cZYdI"}},"required":["apiVersion","kind"],"$id":"rlinf.io.v1alpha1.Cluster"}; +const schema2 = {"type":"object","properties":{"apiVersion":{"$ref":"rgSv5V2hoWAdXQ_XErlwxUiy6phhz4ZEBCjtqeXtVec"},"kind":{"$ref":"KpP0Pr-LdgzYZe_Ch7AbCL6-S9umLrC7dsOL9rmrynU"},"metadata":{"$ref":"P3srTZpWadQH4s4N9KfJR0iCqydQo9YAz7xMB2qf0GQ"},"spec":{"$ref":"YcnFzc_BZQ4f-07lAjG5Dtogv1bwg1MOG1ywYrQrnX0"},"status":{"$ref":"K1c4aJD38nzEc3PCSPTsrApfSSTO_42awjGvqNtX_wk"}},"required":["apiVersion","kind"],"$id":"rlinf.io.v1alpha1.Cluster"}; const schema3 = {"type":"string","enum":["rlinf.io/v1alpha1"]}; function validate3(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ @@ -86,14 +86,17 @@ validate7.errors = vErrors; return errors === 0; } -const schema7 = {"properties":{"agentNamespace":{"$ref":"sLlNkWhANXZ15lvmcujppy1mXxqn6f-GaeQpWRYm_dw"},"cordoned":{"$ref":"CTX4WtafS9K1Gv7k2X1H0J3M9LziUVVRBMOR0rDDzTk"},"draining":{"$ref":"CTX4WtafS9K1Gv7k2X1H0J3M9LziUVVRBMOR0rDDzTk"}},"required":["agentNamespace"],"type":"object","nullable":true}; -const schema8 = {"type":"string"}; +const schema7 = {"properties":{"agentNamespace":{"$ref":"2tG51vkrD8fXpcnHZtqV2HvjVdK98owFTjt5N5f3XEo"},"cordoned":{"$ref":"CTX4WtafS9K1Gv7k2X1H0J3M9LziUVVRBMOR0rDDzTk"},"delisted":{"$ref":"CTX4WtafS9K1Gv7k2X1H0J3M9LziUVVRBMOR0rDDzTk"},"drainPreviousCordoned":{"$ref":"CTX4WtafS9K1Gv7k2X1H0J3M9LziUVVRBMOR0rDDzTk"},"draining":{"$ref":"CTX4WtafS9K1Gv7k2X1H0J3M9LziUVVRBMOR0rDDzTk"}},"required":["agentNamespace"],"type":"object","nullable":true}; +const schema8 = {"minLength":1,"type":"string"}; +import func0 from "@kubernetes-models/validate/runtime/ucs2length"; function validate12(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; -if(typeof data !== "string"){ -const err0 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "string"}}; +const _errs0 = errors; +if(typeof data === "string"){ +if(func0(data) < 1){ +const err0 = {instancePath,schemaPath:"#/minLength",keyword:"minLength",params:{limit: 1}}; if(vErrors === null){ vErrors = [err0]; } @@ -102,7 +105,17 @@ vErrors.push(err0); } errors++; } -const _errs0 = errors; +} +else { +const err1 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "string"}}; +if(vErrors === null){ +vErrors = [err1]; +} +else { +vErrors.push(err1); +} +errors++; +} validate12.errors = vErrors; return errors === 0; } @@ -177,10 +190,10 @@ else { } var valid0 = _errs3 === errors; } -if(data.draining !== undefined){ -let data2 = data.draining; +if(data.delisted !== undefined){ +let data2 = data.delisted; const _errs4 = errors; -if(!(validate0(data2, {instancePath:instancePath+"/draining",parentData:data,parentDataProperty:"draining",rootData}))){ +if(!(validate0(data2, {instancePath:instancePath+"/delisted",parentData:data,parentDataProperty:"delisted",rootData}))){ vErrors = vErrors === null ? validate0.errors : vErrors.concat(validate0.errors); errors = vErrors.length; } @@ -188,18 +201,40 @@ else { } var valid0 = _errs4 === errors; } +if(data.drainPreviousCordoned !== undefined){ +let data3 = data.drainPreviousCordoned; +const _errs5 = errors; +if(!(validate0(data3, {instancePath:instancePath+"/drainPreviousCordoned",parentData:data,parentDataProperty:"drainPreviousCordoned",rootData}))){ +vErrors = vErrors === null ? validate0.errors : vErrors.concat(validate0.errors); +errors = vErrors.length; +} +else { +} +var valid0 = _errs5 === errors; +} +if(data.draining !== undefined){ +let data4 = data.draining; +const _errs6 = errors; +if(!(validate0(data4, {instancePath:instancePath+"/draining",parentData:data,parentDataProperty:"draining",rootData}))){ +vErrors = vErrors === null ? validate0.errors : vErrors.concat(validate0.errors); +errors = vErrors.length; +} +else { +} +var valid0 = _errs6 === errors; +} } validate11.errors = vErrors; return errors === 0; } -const schema9 = {"properties":{"conditions":{"$ref":"hhAxL57cvApnLYfNWQbGuX0ymd4M4buVmG7H0q63I0w"},"lastHeartbeatTime":{"$ref":"YL2AaRx5mZZL3drs48WsIqN7ZQe66kWzKXDNrUwOgoc"},"onlineNodes":{"$ref":"j4SBnKFzNWrJQU4F8rp8qSB0IqehV1dh50NKzJhlTjs"},"phase":{"$ref":"6ehijhkUptDG3qo5MwjI2AutUdnj63qHfu0si557JsY"},"totalNodes":{"$ref":"j4SBnKFzNWrJQU4F8rp8qSB0IqehV1dh50NKzJhlTjs"}},"type":"object","nullable":true}; +const schema9 = {"properties":{"conditions":{"$ref":"hhAxL57cvApnLYfNWQbGuX0ymd4M4buVmG7H0q63I0w"},"lastHeartbeatTime":{"$ref":"YL2AaRx5mZZL3drs48WsIqN7ZQe66kWzKXDNrUwOgoc"},"onlineNodes":{"$ref":"j4SBnKFzNWrJQU4F8rp8qSB0IqehV1dh50NKzJhlTjs"},"phase":{"$ref":"vMczpXMCCY20O_BYEB7ieLvm2vSjEYkG6jYjKiZfaEI"},"totalNodes":{"$ref":"j4SBnKFzNWrJQU4F8rp8qSB0IqehV1dh50NKzJhlTjs"}},"type":"object","nullable":true}; const schema10 = {"items":{"$ref":"dZ13d79yfL0AfKGv8XJr0-7aalgpv7dwgkV5EU9HxwM"},"type":"array","nullable":true}; const schema11 = {"properties":{"lastTransitionTime":{"$ref":"nWP098h5Flk6EOMa_OWYW4EnhyCrisyvgJPgk0WtC9c"},"message":{"$ref":"CG805GD3KzsipQt5EKLOQw9SjU_4PVFnnbOjDh0LzJQ"},"observedGeneration":{"$ref":"Tv0gqme9PlVaT-l831KRjUe4u9reG1m_llVaPqPtqUs"},"reason":{"$ref":"UPBfG4e9zcqG6mx3tN1LWinVzYKOFfIO2VnMgg6lAWI"},"status":{"$ref":"UcCRuQZyUlftnmEwuGDZcciwLAdzzRj4HOLofWAwyW8"},"type":{"$ref":"grctw_GwW6iDVZUzowft1rMz55_d64qiJ_wOfyk2Qoo"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"}; const schema12 = {"format":"date-time","type":"string"}; const formats2 = formats["date-time"]; -function validate20(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate22(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; const _errs0 = errors; @@ -227,14 +262,13 @@ vErrors.push(err1); } errors++; } -validate20.errors = vErrors; +validate22.errors = vErrors; return errors === 0; } const schema13 = {"maxLength":32768,"type":"string"}; -import func0 from "@kubernetes-models/validate/runtime/ucs2length"; -function validate22(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate24(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; const _errs0 = errors; @@ -260,14 +294,14 @@ vErrors.push(err1); } errors++; } -validate22.errors = vErrors; +validate24.errors = vErrors; return errors === 0; } const schema14 = {"format":"int64","type":"integer","minimum":0,"nullable":true}; const formats4 = formats.int64; -function validate24(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate26(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; if((!(((typeof data == "number") && (!(data % 1) && !isNaN(data))) && (isFinite(data)))) && (data !== null)){ @@ -306,14 +340,14 @@ errors++; } if(typeof data === "string"){ } -validate24.errors = vErrors; +validate26.errors = vErrors; return errors === 0; } const schema15 = {"maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"}; const pattern0 = new RegExp("^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", "u"); -function validate26(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate28(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; const _errs0 = errors; @@ -359,13 +393,13 @@ vErrors.push(err3); } errors++; } -validate26.errors = vErrors; +validate28.errors = vErrors; return errors === 0; } const schema16 = {"enum":["True","False","Unknown"],"type":"string"}; -function validate28(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate30(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; if(typeof data !== "string"){ @@ -390,14 +424,14 @@ vErrors.push(err1); } errors++; } -validate28.errors = vErrors; +validate30.errors = vErrors; return errors === 0; } const schema17 = {"maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}; const pattern1 = new RegExp("^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", "u"); -function validate30(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate32(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; const _errs0 = errors; @@ -433,12 +467,12 @@ vErrors.push(err2); } errors++; } -validate30.errors = vErrors; +validate32.errors = vErrors; return errors === 0; } -function validate19(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate21(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; const _errs0 = errors; @@ -496,8 +530,8 @@ errors++; if(data.lastTransitionTime !== undefined){ let data0 = data.lastTransitionTime; const _errs1 = errors; -if(!(validate20(data0, {instancePath:instancePath+"/lastTransitionTime",parentData:data,parentDataProperty:"lastTransitionTime",rootData}))){ -vErrors = vErrors === null ? validate20.errors : vErrors.concat(validate20.errors); +if(!(validate22(data0, {instancePath:instancePath+"/lastTransitionTime",parentData:data,parentDataProperty:"lastTransitionTime",rootData}))){ +vErrors = vErrors === null ? validate22.errors : vErrors.concat(validate22.errors); errors = vErrors.length; } else { @@ -507,8 +541,8 @@ var valid0 = _errs1 === errors; if(data.message !== undefined){ let data1 = data.message; const _errs2 = errors; -if(!(validate22(data1, {instancePath:instancePath+"/message",parentData:data,parentDataProperty:"message",rootData}))){ -vErrors = vErrors === null ? validate22.errors : vErrors.concat(validate22.errors); +if(!(validate24(data1, {instancePath:instancePath+"/message",parentData:data,parentDataProperty:"message",rootData}))){ +vErrors = vErrors === null ? validate24.errors : vErrors.concat(validate24.errors); errors = vErrors.length; } else { @@ -518,8 +552,8 @@ var valid0 = _errs2 === errors; if(data.observedGeneration !== undefined){ let data2 = data.observedGeneration; const _errs3 = errors; -if(!(validate24(data2, {instancePath:instancePath+"/observedGeneration",parentData:data,parentDataProperty:"observedGeneration",rootData}))){ -vErrors = vErrors === null ? validate24.errors : vErrors.concat(validate24.errors); +if(!(validate26(data2, {instancePath:instancePath+"/observedGeneration",parentData:data,parentDataProperty:"observedGeneration",rootData}))){ +vErrors = vErrors === null ? validate26.errors : vErrors.concat(validate26.errors); errors = vErrors.length; } else { @@ -529,8 +563,8 @@ var valid0 = _errs3 === errors; if(data.reason !== undefined){ let data3 = data.reason; const _errs4 = errors; -if(!(validate26(data3, {instancePath:instancePath+"/reason",parentData:data,parentDataProperty:"reason",rootData}))){ -vErrors = vErrors === null ? validate26.errors : vErrors.concat(validate26.errors); +if(!(validate28(data3, {instancePath:instancePath+"/reason",parentData:data,parentDataProperty:"reason",rootData}))){ +vErrors = vErrors === null ? validate28.errors : vErrors.concat(validate28.errors); errors = vErrors.length; } else { @@ -540,8 +574,8 @@ var valid0 = _errs4 === errors; if(data.status !== undefined){ let data4 = data.status; const _errs5 = errors; -if(!(validate28(data4, {instancePath:instancePath+"/status",parentData:data,parentDataProperty:"status",rootData}))){ -vErrors = vErrors === null ? validate28.errors : vErrors.concat(validate28.errors); +if(!(validate30(data4, {instancePath:instancePath+"/status",parentData:data,parentDataProperty:"status",rootData}))){ +vErrors = vErrors === null ? validate30.errors : vErrors.concat(validate30.errors); errors = vErrors.length; } else { @@ -551,8 +585,8 @@ var valid0 = _errs5 === errors; if(data.type !== undefined){ let data5 = data.type; const _errs6 = errors; -if(!(validate30(data5, {instancePath:instancePath+"/type",parentData:data,parentDataProperty:"type",rootData}))){ -vErrors = vErrors === null ? validate30.errors : vErrors.concat(validate30.errors); +if(!(validate32(data5, {instancePath:instancePath+"/type",parentData:data,parentDataProperty:"type",rootData}))){ +vErrors = vErrors === null ? validate32.errors : vErrors.concat(validate32.errors); errors = vErrors.length; } else { @@ -570,12 +604,12 @@ vErrors.push(err5); } errors++; } -validate19.errors = vErrors; +validate21.errors = vErrors; return errors === 0; } -function validate18(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ +function validate20(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){ let vErrors = null; let errors = 0; if((!(Array.isArray(data))) && (data !== null)){ @@ -596,8 +630,8 @@ const len0 = data.length; for(let i0=0; i0 ({ ...item, - clusters: item.clusters.length > 0 ? item.clusters : clusterNames, })); const imageRegistries = [ @@ -540,8 +539,11 @@ export function installMockBackend() { id: name, name, namespace: "kube-system", + clusters: + payload.clusters || + storageClasses.find((item) => item.name === name)?.clusters || + [], provider: payload.provider || "MinIO", - clusters: Array.isArray(payload.clusters) ? payload.clusters : [], endpoint: payload.endpoint || "", region: payload.region || "", bucket: payload.bucket || "", diff --git a/apps/rlark-ui/src/pages/Api.tsx b/apps/rlark-ui/src/pages/Api.tsx index 9342c79..a21a145 100644 --- a/apps/rlark-ui/src/pages/Api.tsx +++ b/apps/rlark-ui/src/pages/Api.tsx @@ -13,6 +13,7 @@ import { type ApiReferenceResponse, } from "../backend"; import type { Copy } from "../i18n"; +import { copyText } from "../utils/clipboard"; export function ApiPage({ copy: c }: { copy: Copy }) { const lang = c.nav.overview === "总览" ? "zh" : "en"; @@ -81,9 +82,8 @@ export function ApiPage({ copy: c }: { copy: Copy }) { }; const copyExample = async (endpoint: ApiReferenceEndpoint) => { - await navigator.clipboard.writeText( - JSON.stringify(endpoint.example, null, 2), - ); + const ok = await copyText(JSON.stringify(endpoint.example, null, 2)); + if (!ok) return; setCopied(true); window.setTimeout(() => setCopied(false), 2000); }; diff --git a/apps/rlark-ui/src/pages/ClusterManagement.tsx b/apps/rlark-ui/src/pages/ClusterManagement.tsx index f2cb8fb..f3abfe8 100644 --- a/apps/rlark-ui/src/pages/ClusterManagement.tsx +++ b/apps/rlark-ui/src/pages/ClusterManagement.tsx @@ -4,6 +4,7 @@ import { Bot, ChevronRight, CloudCog, + MoreHorizontal, MapPin, Network, Server, @@ -28,8 +29,41 @@ import { } from "../components/shared"; import { ColumnFilterPopover } from "../components/ColumnFilterPopover"; import { NodeResourceBrowser } from "../components/NodeResourceBrowser"; +import { clustersApi, nodesApi } from "../backend"; + +type ClusterAction = + "cordon" | "uncordon" | "drain" | "undrain" | "delist" | "restore" | "delete"; + +const clusterActionDescriptions: Record< + Exclude, + { zh: string; en: string } +> = { + cordon: { + zh: "暂停新任务调度到集群", + en: "Stop scheduling new tasks to this cluster", + }, + uncordon: { + zh: "恢复集群的新任务调度", + en: "Resume scheduling new tasks to this cluster", + }, + drain: { + zh: "停止集群内所有运行中的 RLark 任务", + en: "Stop all running RLark tasks in this cluster", + }, + undrain: { + zh: "取消排空并恢复排空前的调度状态", + en: "Cancel draining and restore the pre-drain scheduling state", + }, + delist: { + zh: "下架后用户将无法看到该集群和节点", + en: "Delist: stop showing this cluster and its nodes to users", + }, + restore: { + zh: "恢复集群和节点对用户可见", + en: "Make this cluster and its nodes visible to users again", + }, +}; -// 集群类型中文化：数据层保留英文枚举，渲染时映射 function clusterTypeLabel(type: string, zh: boolean): string { if (!type) return "—"; const map: Record = { @@ -42,91 +76,65 @@ function clusterTypeLabel(type: string, zh: boolean): string { return zh ? entry.zh : entry.en; } -function clusterIDForNode(node: CRDNode) { - return ( - node.metadata.labels?.["rlark.io/cluster-id"] ?? - node.metadata.namespace ?? - "default" - ); -} - -function aggregateClusters( +function aggregateClusterNodes( + cluster: ClusterSummary, nodes: CRDNode[], - clusterTypes = new Map(), workerOnly = false, -): ClusterSummary[] { - const groups = new Map(); - nodes.forEach((node) => { - const id = clusterIDForNode(node); - groups.set(id, [...(groups.get(id) ?? []), node]); - }); - - return [...groups.entries()] - .map(([id, clusterNodes]) => { - const categories: Record = { - cloud: 0, - edge: 0, - robot: 0, - unknown: 0, - }; - clusterNodes.forEach((node) => - getNodeCategories(node).forEach((category) => categories[category]++), +): ClusterSummary { + const clusterNodes = cluster.agentNamespaceConflict + ? [] + : nodes.filter( + (node) => node.metadata.namespace === cluster.agentNamespace, ); - const inferredType = - categories.cloud > 0 && categories.edge === 0 && categories.robot === 0 - ? "Cloud" - : categories.cloud === 0 && - (categories.edge > 0 || categories.robot > 0) - ? "Embodied" - : "Hybrid"; - const type = clusterTypes.get(id) ?? inferredType; - const workerNodes = workerOnly - ? clusterNodes.filter(isBusinessWorkerNode) - : clusterNodes; - const onlineNodes = workerNodes.filter( - (node) => node.status?.phase === "Online", - ).length; - const totalNodes = workerNodes.length; - const phase = - onlineNodes === 0 - ? "Offline" - : onlineNodes === totalNodes - ? "Online" - : "Downgraded"; - const models = (category: NodeCategory) => [ - ...new Set( - clusterNodes - .filter((node) => hasNodeCategory(node, category)) - .map((node) => - category === "cloud" - ? getNodeGPUModel(node) - : getNodeDeviceModel(node), - ) - .filter((value): value is string => Boolean(value)), - ), - ]; - return { - id, - name: id, - type, - region: "", - location: "", - phase, - totalNodes, - onlineNodes, - offlineNodes: totalNodes - onlineNodes, - cloudNodes: categories.cloud, - embodiedNodes: categories.edge, - robots: categories.robot, - gpuModels: models("cloud"), - robotModels: models("robot"), - runningJobs: clusterNodes.filter((node) => - Boolean(node.metadata.labels?.["rlark.io/embodied-task-name"]), - ).length, - description: "", - } satisfies ClusterSummary; - }) - .sort((a, b) => a.name.localeCompare(b.name)); + const categories: Record = { + cloud: 0, + edge: 0, + robot: 0, + unknown: 0, + }; + clusterNodes.forEach((node) => + getNodeCategories(node).forEach((category) => categories[category]++), + ); + const inferredType = + categories.cloud > 0 && categories.edge === 0 && categories.robot === 0 + ? "Cloud" + : categories.cloud === 0 && (categories.edge > 0 || categories.robot > 0) + ? "Embodied" + : "Hybrid"; + const workerNodes = workerOnly + ? clusterNodes.filter(isBusinessWorkerNode) + : clusterNodes; + const onlineNodes = workerNodes.filter( + (node) => node.status?.phase === "Online", + ).length; + const totalNodes = workerNodes.length; + const models = (category: NodeCategory) => [ + ...new Set( + clusterNodes + .filter((node) => hasNodeCategory(node, category)) + .map((node) => + category === "cloud" + ? getNodeGPUModel(node) + : getNodeDeviceModel(node), + ) + .filter((value): value is string => Boolean(value)), + ), + ]; + return { + ...cluster, + type: cluster.type || inferredType, + totalNodes, + onlineNodes, + offlineNodes: totalNodes - onlineNodes, + cloudNodes: categories.cloud, + embodiedNodes: categories.edge, + robots: categories.robot, + gpuModels: models("cloud"), + robotModels: models("robot"), + runningJobs: clusterNodes.filter((node) => + Boolean(node.metadata.labels?.["rlark.io/embodied-task-name"]), + ).length, + }; } function validCount(value: unknown): value is number { @@ -181,9 +189,9 @@ function ClusterStatus({ phase, zh }: { phase: string; zh: boolean }) { Online: { zh: "在线", en: "Online" }, Offline: { zh: "离线", en: "Offline" }, Downgraded: { zh: "服务降级", en: "Downgraded" }, + Delisted: { zh: "已下架", en: "Delisted" }, Cordoned: { zh: "停用调度", en: "Cordoned" }, Draining: { zh: "排空中", en: "Draining" }, - Deleted: { zh: "已删除", en: "Deleted" }, }; const label = labels[phase] ?? { zh: phase, en: phase }; return ( @@ -199,12 +207,14 @@ export function ClusterManagementPage({ onSelectCluster, onSelectNode, workerOnly = false, + adminMode = false, }: { copy: Copy; selectedClusterID?: string; onSelectCluster: (id?: string) => void; onSelectNode: (name: string) => void; workerOnly?: boolean; + adminMode?: boolean; }) { const zh = c.nav.overview === "总览"; const [clusters, setClusters] = useState([]); @@ -216,6 +226,13 @@ export function ClusterManagementPage({ const [phaseFilterValues, setPhaseFilterValues] = useState([]); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(20); + const [actionClusterID, setActionClusterID] = useState(); + const [confirmation, setConfirmation] = useState<{ + cluster: ClusterSummary; + action: ClusterAction; + }>(); + const [actionPending, setActionPending] = useState(false); + const [error, setError] = useState(""); const { openKey, anchorRect, openFor, close } = useColumnFilter(); const fetchClusters = async (isInitial = true) => { @@ -232,33 +249,20 @@ export function ClusterManagementPage({ let resolvedClusters: ClusterSummary[] = []; try { const rawClusters = await clustersApi.list(); - const clusterTypes = new Map( - rawClusters.map((cluster) => [ - cluster.id || cluster.name, - cluster.type, - ]), - ); - const nodeAggregates = new Map( - aggregateClusters(resolvedNodes, clusterTypes, workerOnly).map( - (cluster) => [cluster.id, cluster], - ), - ); resolvedClusters = rawClusters.map((cluster) => normalizeClusterSummary( cluster, - nodeAggregates.get(cluster.id || cluster.name), + aggregateClusterNodes(cluster, resolvedNodes, workerOnly), ), ); } catch { resolvedClusters = []; } - const existingClusterIDs = new Set( - resolvedClusters.map((cluster) => cluster.id || cluster.name), - ); - resolvedNodes = resolvedNodes.filter((node) => - existingClusterIDs.has(clusterIDForNode(node)), - ); - const visibleClusterIDs = new Set(existingClusterIDs); + if (!adminMode) { + resolvedClusters = resolvedClusters.filter( + (cluster) => cluster.phase !== "Delisted", + ); + } if (selectedClusterID) { try { @@ -266,9 +270,11 @@ export function ClusterManagementPage({ ClusterSummary & { nodes?: CRDNode[] } >(selectedClusterID); if (!detail) throw new Error("cluster not found"); - const fullNodes = resolvedNodes.filter( - (node) => clusterIDForNode(node) === selectedClusterID, - ); + const fullNodes = detail.agentNamespaceConflict + ? [] + : resolvedNodes.filter( + (node) => node.metadata.namespace === detail.agentNamespace, + ); const fullNodeByKey = new Map( fullNodes.map((node) => [ `${node.metadata.namespace ?? ""}/${node.metadata.name}`, @@ -290,29 +296,63 @@ export function ClusterManagementPage({ } : node; }); - setDetailNodes(fullNodes.length > 0 ? fullNodes : enrichedDetailNodes); + const detailNodeSource = + fullNodes.length > 0 ? fullNodes : enrichedDetailNodes; + setDetailNodes( + workerOnly + ? detailNodeSource.filter(isBusinessWorkerNode) + : detailNodeSource, + ); if (!resolvedClusters.some((cluster) => cluster.id === detail.id)) { - resolvedClusters = [detail, ...resolvedClusters]; + resolvedClusters = [ + normalizeClusterSummary( + detail, + aggregateClusterNodes(detail, resolvedNodes, workerOnly), + ), + ...resolvedClusters, + ]; } - visibleClusterIDs.add(detail.id || detail.name); } catch { - setDetailNodes( - resolvedNodes.filter( - (node) => clusterIDForNode(node) === selectedClusterID, - ), - ); + setDetailNodes([]); } } else { setDetailNodes([]); } - resolvedClusters = resolvedClusters.filter((cluster) => - visibleClusterIDs.has(cluster.id || cluster.name), - ); setClusters(resolvedClusters); setLoading(false); }; - useAutoRefresh(fetchClusters, 10000, [selectedClusterID]); + useAutoRefresh(fetchClusters, 10000, [selectedClusterID, adminMode]); + + const handleClusterAction = ( + cluster: ClusterSummary, + action: ClusterAction, + ) => { + setError(""); + setActionClusterID(undefined); + setConfirmation({ cluster, action }); + }; + + const confirmClusterAction = async () => { + if (!confirmation) return; + const { cluster, action } = confirmation; + setActionPending(true); + setError(""); + try { + if (action === "delete") { + await clustersApi.remove(cluster.id); + if (selectedClusterID === cluster.id) onSelectCluster(undefined); + } else { + await clustersApi.lifecycle(cluster.id, action); + } + await fetchClusters(false); + setConfirmation(undefined); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setActionPending(false); + } + }; const handleRefresh = async () => { if (refreshing) return; @@ -479,6 +519,11 @@ export function ClusterManagementPage({ onRefresh={handleRefresh} refreshing={refreshing} /> + {error && ( +
+ {error} +
+ )}
- + {adminMode ? (zh ? "操作" : "Actions") : ""}
{loading ? ( @@ -512,11 +557,22 @@ export function ClusterManagementPage({ ? Math.round((cluster.onlineNodes / cluster.totalNodes) * 100) : 0; return ( - + {adminMode ? ( + event.stopPropagation()} + onKeyDown={(event) => event.stopPropagation()} + > + + {actionClusterID === cluster.id && ( + + {cluster.phase === "Delisted" ? ( + + ) : ( + <> + {cluster.phase === "Cordoned" ? ( + + ) : ( + + )} + {cluster.phase === "Draining" ? ( + + ) : ( + + )} + + + + )} + + )} + + ) : ( + + )} +
); }) )} @@ -550,16 +744,27 @@ export function ClusterManagementPage({ label={zh ? "正在刷新集群列表" : "Refreshing cluster list"} /> + {confirmation && ( + setConfirmation(undefined)} + onConfirm={confirmClusterAction} + /> + )} {openKey === "phase" && ( ); } -import { clustersApi, nodesApi } from "../backend"; + +function ClusterActionConfirmDialog({ + cluster, + action, + zh, + pending, + error, + onClose, + onConfirm, +}: { + cluster: ClusterSummary; + action: ClusterAction; + zh: boolean; + pending: boolean; + error: string; + onClose: () => void; + onConfirm: () => Promise; +}) { + useEffect(() => { + const closeOnEscape = (event: KeyboardEvent) => { + if (event.key === "Escape" && !pending) onClose(); + }; + document.addEventListener("keydown", closeOnEscape); + return () => document.removeEventListener("keydown", closeOnEscape); + }, [onClose, pending]); + + const confirmedAction = action === "delete" ? undefined : action; + const content = + action === "delete" + ? { + eyebrow: zh ? "危险操作" : "Danger zone", + title: zh ? "确认删除集群？" : "Delete this cluster?", + description: zh + ? "删除将清理集群及其数据面资源，且无法恢复。" + : "Delete will remove the cluster and its data-plane resources permanently.", + confirm: zh ? "确认删除" : "Delete cluster", + pending: zh ? "删除中…" : "Deleting…", + danger: true, + } + : { + eyebrow: zh ? "集群操作" : "Cluster action", + title: zh ? "确认执行集群操作？" : "Confirm cluster action?", + description: zh + ? clusterActionDescriptions[confirmedAction!].zh + : clusterActionDescriptions[confirmedAction!].en, + confirm: zh ? "确认执行" : "Confirm", + pending: zh ? "处理中…" : "Processing…", + danger: false, + }; + + return ( +
+ event.target === event.currentTarget && !pending && onClose() + } + > +
+
+
+ {content.eyebrow} +

{content.title}

+
+ +
+
+

{content.description}

+
+ {zh ? "目标集群" : "Target cluster"} + {cluster.name} +
+ {error && ( +
+ {error} +
+ )} +
+
+ + +
+
+
+ ); +} diff --git a/apps/rlark-ui/src/pages/Clusters.tsx b/apps/rlark-ui/src/pages/Clusters.tsx index 7f31f78..76cf4f6 100644 --- a/apps/rlark-ui/src/pages/Clusters.tsx +++ b/apps/rlark-ui/src/pages/Clusters.tsx @@ -31,6 +31,7 @@ import type { Copy } from "../i18n"; import type { CRDNode } from "../types"; import type { CRDTask } from "../crdModels"; import { useAutoRefresh } from "../hooks"; +import { copyText } from "../utils/clipboard"; import { categoryLabels, formatResourceQuantity, @@ -44,6 +45,7 @@ import { getNodeResourceSummary, getResourceUsagePercent, isBusinessWorkerNode, + nodeStatusReasonLabel, parseResourceQuantity, } from "../utils/nodes"; import { @@ -362,7 +364,7 @@ export function ClustersPage({
- Distribution + {zh ? "分布" : "Distribution"}

{zh ? "集群分布" : "Cluster Distribution"}

@@ -613,7 +615,10 @@ export function NodeDetailReal({ crName: pod.metadata?.name ?? "", name: pod.spec?.podName || pod.metadata?.name || "—", job, - role: task?.spec?.role ?? "—", + role: + task?.metadata?.annotations?.["rlark.io/ray-role"] ?? + task?.spec?.role ?? + "—", node: node.metadata.name, cluster: pod.metadata?.namespace ?? "default", domain: pod.spec?.domain ?? "", @@ -679,7 +684,7 @@ export function NodeDetailReal({ const formatUsedResource = (key: string) => { const raw = used[key]; if (!raw && (allocatable[key] ?? capacity[key])) { - return formatResourceQuantity(key, "0"); + return formatResourceQuantity(key, "0", zh); } if (raw?.endsWith("%")) { const capacityValue = parseResourceQuantity( @@ -691,10 +696,11 @@ export function NodeDetailReal({ return formatResourceQuantity( key, String((capacityValue * percent) / 100), + zh, ); } } - return formatResourceQuantity(key, raw); + return formatResourceQuantity(key, raw, zh); }; const formatAvailableResource = (key: string) => { const total = parseResourceQuantity(key, allocatable[key] ?? capacity[key]); @@ -707,6 +713,7 @@ export function NodeDetailReal({ return formatResourceQuantity( key, String(Math.max(0, total - (requested ?? 0))), + zh, ); }; const gpuResourceKey = getGPUResourceKey(node); @@ -829,7 +836,7 @@ export function NodeDetailReal({ {node.status?.reason && (
- {node.status.reason} + {nodeStatusReasonLabel(node.status.reason, zh)}
)} @@ -892,23 +899,30 @@ export function NodeDetailReal({ : "Requested"; const usedValue = key === "ephemeral-storage" && diskUsage - ? formatResourceQuantity(key, String(diskUsage.usedBytes)) + ? formatResourceQuantity( + key, + String(diskUsage.usedBytes), + zh, + ) : formatUsedResource(key); const totalValue = key === "ephemeral-storage" && diskUsage ? formatResourceQuantity( key, String(diskUsage.capacityBytes), + zh, ) : formatResourceQuantity( key, allocatable[key] ?? capacity[key], + zh, ); const availableValue = key === "ephemeral-storage" && diskUsage ? formatResourceQuantity( key, String(diskUsage.availableBytes), + zh, ) : formatAvailableResource(key); return ( @@ -1111,7 +1125,9 @@ export function NodeDetailReal({
{deviceResourceKeys.length > 0 ? getNodeDeviceModel(node) || - deviceResourceKeys.map(deviceLabel).join("、") + deviceResourceKeys + .map(deviceLabel) + .join(zh ? "、" : ", ") : zh ? "无端设备" : "No device"} @@ -1289,14 +1305,16 @@ function NodeWorkerTable({ ? `${sshConfig.sshJumpHost}${sshConfig.sshJumpPort ? `:${sshConfig.sshJumpPort}` : ""}` : ""; const sshCommand = sshJump - ? `ssh -J ${sshJump} root@${worker.name}` + ? `ssh -J ${sshJump} root@${worker.name} -p 2222` : ""; const isExpanded = expanded === worker.id; return ( - {worker.name} + + {worker.name} +
- {rr.mounts.map((mount, index) => ( -
-
{ - const next = - mount.type === "storage" - ? "host" - : "storage"; - updateRRMount(role, index, "type", next); - if (next === "storage") { - fetchStorageClasses(rr.cluster); - } - }} - > + {rr.mounts.map((mount, index) => + mount.type === "opaque" ? ( + // 表单无法编辑的卷（emptyDir、configMap 等）只读展示， + // 提交时按原始定义原样写回。 +
+
+ +
+ + -
+ ) : ( +
+
{ + const next = + mount.type === "storage" + ? "host" + : "storage"; + updateRRMount(role, index, "type", next); + if (next === "storage") { + fetchStorageClasses(rr.cluster); + } }} > - {zh ? "对象存储" : "Object storage"} - -
-
+ + + {mount.type === "storage" && ( + - )} - -
- ))} + onBlur={(e) => { + const value = Number(e.target.value); + if ( + isNaN(value) || + value < 1 || + value > 200 + ) { + // 恢复默认值 + updateRRMount( + role, + index, + "pvcSizeGb", + 10, + ); + // 显示错误提示 + setError( + zh + ? "存储大小必须在 1 到 200 Gi 之间" + : "PVC size must be between 1 and 200 Gi", + ); + setErrorNonce((n) => n + 1); + } + }} + /> + + )} + +
+ ), + )} @@ -1674,19 +1737,37 @@ export function CreateJobModal({
- {roles.map((entry) => ( - - ))} + {roles.map((entry) => { + const headerAllowed = canBeHeader(entry.id); + const isHeader = effectiveHeader === entry.id; + return ( + + ); + })}
@@ -1751,6 +1832,13 @@ export function CreateJobModal({ : "No SSH keys found. Add keys in the SSH Key Management page first."} )} + {droppedSSHKeyCount > 0 && ( + + {zh + ? `已移除 ${droppedSSHKeyCount} 个公钥：其用户记录已从 SSH 公钥管理中删除，不会注入到新任务。` + : `Removed ${droppedSSHKeyCount} public key(s): their owner records were deleted from SSH key management and will not be injected into the new job.`} + + )} {sshPublicKey && ( {zh diff --git a/apps/rlark-ui/src/pages/Domains.tsx b/apps/rlark-ui/src/pages/Domains.tsx index 9f20886..a8c9c4b 100644 --- a/apps/rlark-ui/src/pages/Domains.tsx +++ b/apps/rlark-ui/src/pages/Domains.tsx @@ -207,16 +207,14 @@ export function DomainsPage({ />
{showCreate && ( -
- e.target === e.currentTarget && setShowCreate(false) - } - > + // 表单内容无法恢复，遮罩点击不关闭弹层；关闭入口为右上角按钮和取消按钮。 +
- NEW DOMAIN + + {zh ? "新建网络域" : "NEW DOMAIN"} +

{zh ? "创建网络域" : "Create Domain"}

{workerRoles.map((role) => (
); - // 全屏时挂到 body 下，避免侧边栏/顶栏的 stacking context - // （半透明 + backdrop-filter）与全屏终端叠出脏视觉。 + // 全屏时挂到 app shell 下，避免侧边栏/顶栏的 stacking + // context（半透明 + backdrop-filter）与全屏终端叠出脏视觉。 return logFullscreen - ? createPortal(terminalNode, document.body) + ? createPortal(terminalNode, overlayContainer()) : terminalNode; })() )} @@ -3480,7 +3419,7 @@ export function JobDetailPage({ onClose={workerColumnFilter.close} zh={zh} />, - document.body, + overlayContainer(), )} {workerColumnFilter.openKey === "phase" && typeof document !== "undefined" && @@ -3494,7 +3433,7 @@ export function JobDetailPage({ onClose={workerColumnFilter.close} zh={zh} />, - document.body, + overlayContainer(), )} {workerColumnFilter.openKey === "cluster" && typeof document !== "undefined" && @@ -3508,7 +3447,7 @@ export function JobDetailPage({ onClose={workerColumnFilter.close} zh={zh} />, - document.body, + overlayContainer(), )} {workerColumnFilter.openKey === "kind" && typeof document !== "undefined" && @@ -3522,7 +3461,7 @@ export function JobDetailPage({ onClose={workerColumnFilter.close} zh={zh} />, - document.body, + overlayContainer(), )}
); @@ -3734,20 +3673,35 @@ function JobPublicOverview({ {resolvedSSHKeys.length > 0 ? (
    {resolvedSSHKeys.map(({ publicKey, owners }, index) => { + const ownerRemoved = owners.length === 0 && sshKeysLoaded; const ownerLabel = owners.length > 0 ? owners.map(({ user }) => user).join(", ") : sshKeysLoaded ? zh - ? "未知用户" - : "Unknown user" + ? "用户已删除" + : "User deleted" : zh ? "加载用户中…" : "Loading user…"; + const ownerTitle = ownerRemoved + ? zh + ? "该公钥的用户记录已从 SSH 公钥管理中删除，公钥仍注入在运行中的 Pod 内" + : "The owner record was removed from SSH key management; the key stays injected in running pods" + : ownerLabel; return (
  • - {ownerLabel} + + {ownerLabel} + {publicKey}
    • ); @@ -3827,19 +3781,13 @@ function JobPublicOverview({ zh={zh} onClose={() => setTagPopover(null)} />, - document.body, + overlayContainer(), )} {tagsEditing && typeof document !== "undefined" && createPortal( -
    - event.target === event.currentTarget && - !tagsSaving && - cancelTagsEdit() - } - > + // 标签编辑内容无法恢复，遮罩点击不关闭弹层；关闭入口为右上角按钮和取消按钮。 +
    , - document.querySelector(".app-shell") ?? document.body, + overlayContainer(), )}
@@ -3983,6 +3931,7 @@ function RoleRuntimeConfig({ roles, selectedRole, onRoleChange, + storageClassDisplayNames, nodeDeviceModelMap, }: { job: Job; @@ -3990,13 +3939,20 @@ function RoleRuntimeConfig({ roles: string[]; selectedRole: string; onRoleChange: (role: string) => void; + storageClassDisplayNames: Record; nodeDeviceModelMap: Record< string, { gpuModel?: string; deviceModel?: string } >; }) { const zh = c.nav.overview === "总览"; + const [imageCopied, setImageCopied] = useState(false); const resource = job.resources.find((item) => item.role === selectedRole); + const handleCopyImage = async () => { + if (!(await copyText(resource?.image ?? ""))) return; + setImageCopied(true); + window.setTimeout(() => setImageCopied(false), 1600); + }; const taskName = resource ? taskResourceName(job.name, resource.role) : ""; const taskStatus = job.taskStatuses.find( (status) => @@ -4082,12 +4038,31 @@ function RoleRuntimeConfig({ {zh ? "镜像" : "Image"} {resource.image || "—"}
@@ -4154,18 +4129,31 @@ function RoleRuntimeConfig({ zh ? "大小" : "Size", zh ? "挂载到 Worker" : "Mount in worker", ]} - rows={resource.mounts.map((mount) => [ - mount.type === "storage" - ? zh - ? "对象存储" - : "Object storage" - : zh - ? "主机目录" - : "Host directory", - mount.type === "storage" ? mount.objectStorage : mount.hostPath, - mount.type === "storage" ? `${mount.pvcSizeGb} Gi` : "", - mount.mountPath, - ])} + rows={resource.mounts.map((mount) => { + if (mount.type === "opaque") { + return [ + zh ? "其他卷类型" : "Other volume", + describeVolumeSource(mount.rawVolume), + "", + mount.mountPath, + ]; + } + return [ + mount.type === "storage" + ? zh + ? "对象存储" + : "Object storage" + : zh + ? "主机目录" + : "Host directory", + mount.type === "storage" + ? (storageClassDisplayNames[mount.objectStorage] ?? + mount.objectStorage) + : mount.hostPath, + mount.type === "storage" ? `${mount.pvcSizeGb} Gi` : "", + mount.mountPath, + ]; + })} empty={zh ? "未配置数据挂载" : "No data mounts configured"} />
@@ -4234,11 +4222,26 @@ function formatWorkerCreatedAt(startedAt: string, index: number) { return formatChinaDateTime(date.toISOString()); } -function getNodeKindLabel(worker: WorkerItem) { +// Node kind is derived from worker node/role naming. getNodeKind returns a +// stable key so column filters keep matching across language switches, while +// nodeKindLabel renders the localized text. +const nodeKindLabels: Record = { + robot: { zh: "具身真机", en: "Embodied robot" }, + edge: { zh: "具身算力", en: "Embodied compute" }, + cloud: { zh: "云算力", en: "Cloud compute" }, +}; + +function getNodeKind(worker: WorkerItem) { const value = `${worker.node} ${worker.role}`.toLowerCase(); - if (value.includes("robot")) return "具身真机"; - if (value.includes("edge") || value.includes("camera")) return "具身算力"; - return "云算力"; + if (value.includes("robot")) return "robot"; + if (value.includes("edge") || value.includes("camera")) return "edge"; + return "cloud"; +} + +function nodeKindLabel(kind: string, zh: boolean) { + const label = nodeKindLabels[kind]; + if (!label) return kind; + return zh ? label.zh : label.en; } function exportLogs( @@ -4428,7 +4431,7 @@ function MetricsDashboard({
{metricCards.map((metric) => ( - + ))}
@@ -4440,11 +4443,13 @@ function TimeSeriesCard({ unit, color, values, + zh, }: { label: string; unit: string; color: string; values: number[]; + zh: boolean; }) { const max = Math.max(...values); const min = Math.min(...values); @@ -4470,7 +4475,7 @@ function TimeSeriesCard({ @@ -4480,7 +4485,7 @@ function TimeSeriesCard({
-60m -30m - now + {zh ? "现在" : "now"}
); @@ -4596,6 +4601,20 @@ function workerEventReasonLabel( return zh ? "等待 Worker 启动" : "Waiting for Worker startup"; } +// workerEventLabel 先把 kubelet 的通用 Failed 事件按 message 收敛到更精确的 +// reason，再取展示文案：否则"镜像拉取失败"会一律显示成"Worker 启动失败"。 +function workerEventLabel( + event: NodeEventEntry, + zh: boolean, + failed = false, +): string { + return workerEventReasonLabel( + refineWorkerEventReason(event.reason, event.message), + zh, + failed, + ); +} + function jobFailureMessage(message: string | undefined, zh: boolean): string { if (!message) return zh ? "Worker 运行失败" : "Worker failed"; if (/oomkilled|out of memory/i.test(message)) { @@ -4636,6 +4655,7 @@ function jobFailureMessage(message: string | undefined, zh: boolean): string { export function PullProgressInfo({ progress, events = [], + resolvedEvents = [], zh, emptyMessage, statusMessage, @@ -4646,6 +4666,7 @@ export function PullProgressInfo({ }: { progress: PullProgressEntry[]; events?: NodeEventEntry[]; + resolvedEvents?: NodeEventEntry[]; zh: boolean; emptyMessage?: string; statusMessage?: string; @@ -4664,20 +4685,25 @@ export function PullProgressInfo({ above: boolean; arrowLeft: number; } | null>(null); - const recentEvents = [ - ...new Map( - [...events] - .sort((left, right) => { - const leftTime = Date.parse(left.lastTime ?? "") || 0; - const rightTime = Date.parse(right.lastTime ?? "") || 0; - return rightTime - leftTime; - }) - .map((event) => [ - workerEventReasonLabel(event.reason, zh, failed), - event, - ]), - ).values(), - ].slice(0, 4); + // 按展示文案去重后取最近 4 条，避免 tooltip 被重复 reason 撑爆。同一 reason + // 会被反复上报，这里保留 lastTime 最新的一条，再整体按时间倒序。注意不能用 + // new Map(pairs) 直接去重：重复 key 会保留首次出现的位置但取最后一次出现的 + // 值，先排序再去重就会得到"位置是最新、时间戳是最旧"的错乱结果。 + const summarizeEvents = (entries: NodeEventEntry[]) => { + const newest = new Map(); + for (const event of entries) { + const key = workerEventLabel(event, zh, failed); + const existing = newest.get(key); + if (!existing || workerEventTime(event) >= workerEventTime(existing)) { + newest.set(key, event); + } + } + return [...newest.values()] + .sort((left, right) => workerEventTime(right) - workerEventTime(left)) + .slice(0, 4); + }; + const recentEvents = summarizeEvents(events); + const recentResolvedEvents = summarizeEvents(resolvedEvents); const measure = () => { const icon = wrapperRef.current; @@ -4689,11 +4715,19 @@ export function PullProgressInfo({ const gap = 10; const margin = 12; - const spaceAbove = iconRect.top; + // 顶栏是 position: sticky 且始终覆盖视口顶部，它占住的高度不是可用空间。 + // 不扣掉它的话，详情页顶部徽章的 tooltip 会被放到顶栏底下而被遮挡。 + const topbar = document.querySelector(".topbar"); + const topInset = topbar ? topbar.getBoundingClientRect().bottom : 0; + const spaceAbove = iconRect.top - topInset; const spaceBelow = window.innerHeight - iconRect.bottom; const above = spaceAbove >= tooltipH + gap || spaceAbove >= spaceBelow; - const top = above ? iconRect.top - tooltipH - gap : iconRect.bottom + gap; + let top = above ? iconRect.top - tooltipH - gap : iconRect.bottom + gap; + top = Math.max( + topInset + margin, + Math.min(top, window.innerHeight - tooltipH - margin), + ); const iconCenter = iconRect.left + iconRect.width / 2; let left = iconCenter - tooltipW / 2; @@ -4890,7 +4924,33 @@ export function PullProgressInfo({ : "event-pending" }`} > - {workerEventReasonLabel(ev.reason, zh, failed)} + {workerEventLabel(ev, zh, failed)} + + {ev.lastTime && ( + + {formatChinaDateTime(ev.lastTime)} + + )} + + ))} + + )} + {recentResolvedEvents.length > 0 && ( + <> + + {zh ? "历史事件" : "Earlier Events"} + {resolvedEvents.length > recentResolvedEvents.length && ( + + {zh + ? `最近 ${recentResolvedEvents.length} 条` + : `Latest ${recentResolvedEvents.length}`} + + )} + + {recentResolvedEvents.map((ev, i) => ( + + + {workerEventLabel(ev, zh)} {ev.lastTime && ( @@ -4902,7 +4962,7 @@ export function PullProgressInfo({ )} , - document.body, + overlayContainer(), )} ); @@ -4953,11 +5013,8 @@ function WorkerTableRow({ const sshJump = sshConfig?.jumpHost ? `${sshConfig.jumpHost}${sshConfig.jumpPort ? ":" + sshConfig.jumpPort : ""}` : ""; - // head 节点直连 Pod 的 22 端口；非 head 节点通过节点 sshd 的 2222 端口转发到目标 Pod。 const sshCommand = sshJump - ? isHeader - ? `ssh -J ${sshJump} root@${worker.name}` - : `ssh -J ${sshJump} root@${worker.name} -p 2222` + ? `ssh -J ${sshJump} root@${worker.name} -p 2222` : ""; const handleCopy = async () => { if (!(await copyText(sshCommand))) return; @@ -5038,10 +5095,13 @@ function WorkerTableRow({ (worker.phase === "Pending" || worker.phase === "Failed" || (worker.pullProgress && worker.pullProgress.length > 0) || - (worker.events && worker.events.length > 0)) && ( + (worker.events && worker.events.length > 0) || + (worker.resolvedEvents && + worker.resolvedEvents.length > 0)) && ( - {getNodeKindLabel(worker)} + + {nodeKindLabel(getNodeKind(worker), zh)} + {getNodeRank(worker, pods)} diff --git a/apps/rlark-ui/src/pages/Login.tsx b/apps/rlark-ui/src/pages/Login.tsx index 476abc0..34caf5e 100644 --- a/apps/rlark-ui/src/pages/Login.tsx +++ b/apps/rlark-ui/src/pages/Login.tsx @@ -2,12 +2,16 @@ import { useEffect, useState, type FormEvent } from "react"; import { AlertCircle, ArrowRight, Eye, EyeOff } from "lucide-react"; import { storeAuthSession } from "../api"; import { authApi } from "../backend"; +import type { Lang } from "../i18n"; export function UserLogin({ onLogin, + lang, }: { onLogin: (username: string) => void; + lang: Lang; }) { + const zh = lang === "zh"; const [username, setUsername] = useState("user"); const [password, setPassword] = useState(""); const [showPassword, setShowPassword] = useState(false); @@ -23,7 +27,7 @@ export function UserLogin({ const handleSubmit = (e: FormEvent) => { e.preventDefault(); if (!username.trim() || !password.trim()) { - setError("请输入账号和密码"); + setError(zh ? "请输入账号和密码" : "Enter your username and password"); return; } setLoading(true); @@ -32,7 +36,7 @@ export function UserLogin({ .login(username.trim(), password) .then((result) => { if (!result.token || result.role !== "user") { - throw new Error("登录响应无效"); + throw new Error(zh ? "登录响应无效" : "Invalid login response"); } storeAuthSession(result.token, result.role); sessionStorage.setItem("rlark-user-auth", "1"); @@ -40,7 +44,13 @@ export function UserLogin({ onLogin(username.trim()); }) .catch((err) => { - setError(err.status === 401 ? "账号或密码错误" : err.message); + setError( + err.status === 401 + ? zh + ? "账号或密码错误" + : "Incorrect username or password" + : err.message, + ); setLoading(false); }); }; @@ -55,16 +65,22 @@ export function UserLogin({
RLark 具身智能云原生纳管平台
-

用户登录

-

请输入账号和密码

+

{zh ? "用户登录" : "User sign-in"}

+

+ {zh ? "请输入账号和密码" : "Enter your username and password"} +

- +
- +
- 管理后台 + {zh ? "管理后台" : "Admin console"}
diff --git a/apps/rlark-ui/src/pages/Overview.tsx b/apps/rlark-ui/src/pages/Overview.tsx index 6c66126..4970773 100644 --- a/apps/rlark-ui/src/pages/Overview.tsx +++ b/apps/rlark-ui/src/pages/Overview.tsx @@ -13,11 +13,14 @@ import type { Copy } from "../i18n"; import type { CRDNode, Page, ResourceRow } from "../types"; import { useAutoRefresh } from "../hooks"; import { crdToJob } from "../utils/crd"; +import { effectiveJobPhase, isActiveJob } from "../utils/jobPhase"; import { getNodeCategories, getNodeDeviceModel, getNodeGPUModel, hasNodeCategory, + isBusinessWorkerNode, + nodeStatusReasonLabel, } from "../utils/nodes"; import { MetricCard, @@ -67,7 +70,7 @@ export function Overview({ } }; - const displayNodes = realNodes; + const displayNodes = realNodes.filter(isBusinessWorkerNode); const embodiedClusters = realClusters.filter((x) => x.type === "Embodied"); const runningJobs = realJobs.filter((x) => x.phase === "Running").length; const gpuModelList = Array.from( @@ -151,6 +154,8 @@ export function Overview({ ]; const robotNodes = displayNodes.filter((n) => hasNodeCategory(n, "robot")); + // 「活跃任务」面板只展示非终态任务（等待中 / 运行中 / 停止中）。 + const activeJobs = realJobs.filter(isActiveJob); return (
@@ -299,29 +305,40 @@ export function Overview({
- {realJobs.slice(0, 4).map((job) => ( - - ))} + {activeJobs.length === 0 ? ( +

+ {c.overview.noLiveJobs} +

+ ) : ( + activeJobs.slice(0, 4).map((job) => ( + + )) + )}
diff --git a/apps/rlark-ui/src/pages/SSHKeys.tsx b/apps/rlark-ui/src/pages/SSHKeys.tsx index f304b14..a734de4 100644 --- a/apps/rlark-ui/src/pages/SSHKeys.tsx +++ b/apps/rlark-ui/src/pages/SSHKeys.tsx @@ -10,6 +10,7 @@ import { } from "lucide-react"; import type { Copy } from "../i18n"; import { formatChinaDateTime } from "../utils/time"; +import { copyText } from "../utils/clipboard"; import { findSSHKeyDuplicate } from "../utils/sshKeys"; import { ColumnFilterButton, @@ -170,12 +171,18 @@ export function SSHKeysPage({ ? `ssh -J ${sshConfig.sshJumpHost}${sshConfig.sshJumpPort ? ":" + sshConfig.sshJumpPort : ""} root@` : `ssh -J : root@`; - const handleCopy = () => { - navigator.clipboard.writeText(sshCommand); + const handleCopy = async () => { + if (!(await copyText(sshCommand))) return; setCopied(true); setTimeout(() => setCopied(false), 2000); }; + const handleCopyKey = async (id: string, publicKey: string) => { + if (!(await copyText(publicKey))) return; + setCopiedKey(id); + setTimeout(() => setCopiedKey(""), 2000); + }; + return (
@@ -254,8 +261,21 @@ export function SSHKeysPage({ {sshCommand}
-
- - - -
- {zh ? "关联集群" : "Linked clusters"} - {associatedClusters.size} - {zh ? "个集群已接入存储" : "clusters with storage"} -
-
@@ -286,8 +297,8 @@ export function StorageClassesPage({ {zh ? "存储资源" : "Storage resources"} {zh - ? "管理存储桶、集群关联和访问配置" - : "Manage buckets, cluster links and access settings"} + ? "管理存储桶和访问配置" + : "Manage buckets and access settings"}
@@ -306,13 +317,6 @@ export function StorageClassesPage({ /> {c.storageClass.bucket} - - - {c.storageClass.description} {zh ? "操作" : "Actions"} @@ -320,7 +324,7 @@ export function StorageClassesPage({ {filtered.length === 0 && ( - + {loading ? zh ? "加载中..." @@ -351,15 +355,6 @@ export function StorageClassesPage({ {sc.bucket} - -
- {sc.clusters.length > 0 - ? sc.clusters.map((cluster) => ( - {cluster} - )) - : "—"} -
- {sc.description || "—"} @@ -372,16 +367,7 @@ export function StorageClassesPage({ title={c.files.eyebrow} onClick={(e) => { e.stopPropagation(); - const cluster = sc.clusters[0] || ""; - if (!cluster) { - alert( - zh - ? "该存储类未关联集群" - : "This storage class has no associated cluster", - ); - return; - } - onBrowseFiles(cluster, sc.id); + onBrowseFiles(sc.id); }} > @@ -435,19 +421,6 @@ export function StorageClassesPage({ zh={zh} /> )} - {columnFilter.openKey === "cluster" && ( - ({ value: cl, label: cl }))} - selected={clusterFilter} - onChange={setClusterFilter} - anchorRect={columnFilter.anchorRect} - onClose={columnFilter.close} - zh={zh} - /> - )} {editingClass && (

- {zh - ? "访问端点、集群范围与连接方式" - : "Endpoint, cluster scope and connection mode"} + {zh ? "访问端点与连接方式" : "Endpoint and connection mode"}

@@ -563,10 +534,6 @@ export function StorageClassDetailPage({ : "Disabled"}
-
- {c.storageClass.clusters} - {storageClass.clusters.join(", ") || "—"} -
{c.storageClass.description} {storageClass.description || "—"} @@ -593,40 +560,36 @@ export function StorageClassCreatePage({ const isEdit = Boolean(initialStorageClass); const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(""); - const [clusterOptions, setClusterOptions] = useState( - mockClusters.map((cluster) => ({ - id: cluster.name, - name: cluster.name, - description: cluster.region, - })), - ); const [form, setForm] = useState(() => storageClassToForm(initialStorageClass), ); + const [providerOptions, setProviderOptions] = useState([]); + const [clusterOptions, setClusterOptions] = useState< + Array<{ id: string; name: string }> + >([]); + const [clustersMenuOpen, setClustersMenuOpen] = useState(false); + const clustersMenuRef = useRef(null); useEffect(() => { setForm(storageClassToForm(initialStorageClass)); }, [initialStorageClass]); + // 提供商目录来自后端，取值必须与后端校验的 canonical 值一致，否则下发到 + // 数据面的 rclone 配置无法识别。 useEffect(() => { let cancelled = false; - clustersApi - .list<{ - id?: string; - name?: string; - region?: string; - location?: string; - }>() + storageClassesApi + .providers() .then((data) => { if (cancelled) return; - const options: ClusterOption[] = data - .map((cluster) => ({ - id: cluster.id ?? cluster.name ?? "", - name: cluster.name ?? cluster.id ?? "", - description: cluster.region ?? cluster.location ?? "", - })) - .filter((cluster: ClusterOption) => cluster.id || cluster.name); - if (options.length > 0) setClusterOptions(options); + setProviderOptions( + data + .map((provider) => ({ + name: provider.name ?? provider.value ?? "", + value: provider.value ?? "", + })) + .filter((provider) => provider.value), + ); }) .catch(() => {}); return () => { @@ -634,22 +597,58 @@ export function StorageClassCreatePage({ }; }, []); + useEffect(() => { + let cancelled = false; + clustersApi + .list<{ id?: string; name?: string }>() + .then((clusters) => { + if (cancelled) return; + setClusterOptions( + clusters + .map((cluster) => { + const id = cluster.id || cluster.name || ""; + return { id, name: cluster.name || id }; + }) + .filter((cluster) => cluster.id), + ); + }) + .catch(() => { + if (!cancelled) setClusterOptions([]); + }); + return () => { + cancelled = true; + }; + }, []); + + useEffect(() => { + const handlePointerDown = (event: PointerEvent) => { + if ( + clustersMenuRef.current && + !clustersMenuRef.current.contains(event.target as Node) + ) { + setClustersMenuOpen(false); + } + }; + document.addEventListener("pointerdown", handlePointerDown); + return () => document.removeEventListener("pointerdown", handlePointerDown); + }, []); + useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { - if (event.key === "Escape" && !submitting) onBack(); + if (event.key === "Escape") { + if (clustersMenuOpen) { + setClustersMenuOpen(false); + return; + } + if (!submitting) onBack(); + } }; document.addEventListener("keydown", handleKeyDown); return () => document.removeEventListener("keydown", handleKeyDown); - }, [onBack, submitting]); + }, [clustersMenuOpen, onBack, submitting]); const handleSubmit = async (e: FormEvent) => { e.preventDefault(); - if (form.clusters.length === 0) { - setError( - zh ? "请至少选择一个关联集群。" : "Select at least one linked cluster.", - ); - return; - } if (!isEdit && !form.access_key_secret.trim()) { setError( zh ? "Access Key Secret 不能为空。" : "Access Key Secret is required.", @@ -664,20 +663,18 @@ export function StorageClassCreatePage({ else await storageClassesApi.create(form); onCreated?.(); onBack(); - } catch { - setError(c.storageClass.createFailed); + } catch (err) { + setError( + storageClassSubmissionError(err, zh, c.storageClass.createFailed), + ); } finally { setSubmitting(false); } }; return ( -
{ - if (event.target === event.currentTarget && !submitting) onBack(); - }} - > + // 表单内容无法恢复，遮罩点击不关闭弹层；关闭入口为右上角按钮、取消按钮和 Esc。 +
{isEdit ? zh - ? "调整对象存储配置、访问参数和关联集群。" - : "Update object storage settings, access parameters and linked clusters." + ? "调整对象存储配置和访问参数。" + : "Update object storage settings and access parameters." : c.storageClass.desc}

@@ -730,6 +727,7 @@ export function StorageClassCreatePage({ -
+
+
+ {zh ? "关联集群" : "Associated clusters"} +
+ + {clustersMenuOpen && ( +
+ {Array.from( + new Set([ + ...clusterOptions.map((cluster) => cluster.id), + ...form.clusters, + ]), + ) + .sort() + .map((id) => { + const option = clusterOptions.find( + (cluster) => cluster.id === id, + ); + return ( + + ); + })} + {clusterOptions.length === 0 && + form.clusters.length === 0 && ( +

{zh ? "暂无可选集群" : "No clusters available"}

+ )} +
+ )}
@@ -801,12 +867,19 @@ export function StorageClassCreatePage({
- {error &&

{error}

} + {error && ( +
+
+ )}
- {open && ( -
-
- - setQuery(event.target.value)} - placeholder={ - zh - ? "搜索集群名称、ID 或地域" - : "Search cluster name, ID or region" - } - /> - - {zh - ? `${filteredOptions.length}/${options.length} 个` - : `${filteredOptions.length}/${options.length}`} - -
- -
- {filteredOptions.map((option) => ( - - ))} - {filteredOptions.length === 0 && ( -

{zh ? "没有匹配的集群" : "No matching clusters"}

- )} -
-
- )} -
- ); -} - export function StorageClassFilesPage({ copy: c, sub, @@ -1032,9 +975,7 @@ export function StorageClassFilesPage({ onBack: () => void; }) { const zh = c === copy.zh; - const parts = sub.split("/"); - const cluster = decodeURIComponent(parts[0] || ""); - const name = decodeURIComponent(parts[1] || ""); + const name = decodeURIComponent(sub); const [prefix, setPrefix] = useState(""); const [objects, setObjects] = useState([]); const [commonPrefixes, setCommonPrefixes] = useState([]); @@ -1047,11 +988,9 @@ export function StorageClassFilesPage({ const [pageSize, setPageSize] = useState(20); const fetchFiles = async () => { - if (!cluster || !name) { + if (!name) { setError( - zh - ? "参数错误：缺少集群或存储类名称" - : "Error: missing cluster or storage class name", + zh ? "参数错误：缺少存储类名称" : "Error: missing storage class name", ); return; } @@ -1061,7 +1000,7 @@ export function StorageClassFilesPage({ const data = await storageObjectsApi.list<{ objects?: any[]; common_prefixes?: string[]; - }>(name, cluster, prefix); + }>(name, prefix); setObjects(data.objects || []); setCommonPrefixes(data.common_prefixes || []); } catch (e: any) { @@ -1075,10 +1014,10 @@ export function StorageClassFilesPage({ useEffect(() => { fetchFiles(); - }, [cluster, name, prefix]); + }, [name, prefix]); const handleUpload = async (files: FileList) => { - if (!cluster || !name) return; + if (!name) return; setUploading(true); setError(""); try { @@ -1090,7 +1029,7 @@ export function StorageClassFilesPage({ setUploadProgress( `${zh ? "上传中" : "Uploading"}: ${file.name} (${i + 1}/${files.length})`, ); - await storageObjectsApi.upload(name, cluster, formData); + await storageObjectsApi.upload(name, formData); } setUploadProgress(""); fetchFiles(); @@ -1105,7 +1044,6 @@ export function StorageClassFilesPage({ try { const data = await storageObjectsApi.download<{ url?: string }>( name, - cluster, key, ); if (data.url) { @@ -1122,21 +1060,28 @@ export function StorageClassFilesPage({ if (!confirm(zh ? `确定删除文件 "${key}" 吗?` : `Delete file "${key}"?`)) return; try { - await storageObjectsApi.remove(name, cluster, key); + await storageObjectsApi.remove(name, key); fetchFiles(); } catch (e: any) { alert(e?.message || (zh ? "删除失败" : "Delete failed")); } }; + const normalizeDirectoryPrefix = (directoryPrefix: string) => { + const normalized = directoryPrefix.replace(/^\/+|\/+$/g, ""); + return normalized ? `${normalized}/` : ""; + }; + const navigateFolder = (folderPrefix: string) => { - setPrefix(folderPrefix); + setPrefix(normalizeDirectoryPrefix(folderPrefix)); }; const goUp = () => { - if (!prefix) return; - const lastSlash = prefix.lastIndexOf("/"); - setPrefix(lastSlash >= 0 ? prefix.slice(0, lastSlash + 1) : ""); + const currentDirectory = normalizeDirectoryPrefix(prefix).slice(0, -1); + const parentEnd = currentDirectory.lastIndexOf("/"); + setPrefix( + parentEnd >= 0 ? `${currentDirectory.slice(0, parentEnd + 1)}` : "", + ); }; const filteredObjects = objects.filter((obj) => @@ -1178,8 +1123,13 @@ export function StorageClassFilesPage({ let path = ""; for (const part of parts) { path += part + "/"; + const breadcrumbPath = path; crumbs.push( - , ); @@ -1197,11 +1147,9 @@ export function StorageClassFilesPage({

{name || c.files.title}

- {c.files.cluster}: {cluster || "—"} · {c.files.storageClass}:{" "} - {name || "—"} + {c.files.storageClass}: {name || "—"}

- {cluster || "—"} {prefix ? zh @@ -1232,13 +1180,6 @@ export function StorageClassFilesPage({ {zh ? "存储类" : "Storage class"} {name || "—"}
-
- - - - {zh ? "所属集群" : "Cluster"} - {cluster || "—"} -
@@ -1469,3 +1410,4 @@ export function StorageClassFilesPage({ ); } import { clustersApi, storageClassesApi, storageObjectsApi } from "../backend"; +import { ApiError, errorMessage } from "../api"; diff --git a/apps/rlark-ui/src/pages/SystemConfig.tsx b/apps/rlark-ui/src/pages/SystemConfig.tsx index c98ede1..bd6119f 100644 --- a/apps/rlark-ui/src/pages/SystemConfig.tsx +++ b/apps/rlark-ui/src/pages/SystemConfig.tsx @@ -17,6 +17,7 @@ import { type SystemConfigResponse, } from "../backend"; import { resolveDeploymentConfig } from "../utils/deployYaml"; +import { copyText } from "../utils/clipboard"; interface LogBackendConfig { endpoint: string; @@ -228,7 +229,7 @@ export function SystemConfigPage({ copy: c }: { copy: Copy }) { : ""; const copySSHCommand = async () => { - await navigator.clipboard.writeText(sshCommand); + if (!(await copyText(sshCommand))) return; setCopied(true); window.setTimeout(() => setCopied(false), 2000); }; @@ -703,12 +704,16 @@ export function SystemConfigPage({ copy: c }: { copy: Copy }) { }} > - + diff --git a/apps/rlark-ui/src/pages/Workflows.tsx b/apps/rlark-ui/src/pages/Workflows.tsx index a5a7908..efcd3ea 100644 --- a/apps/rlark-ui/src/pages/Workflows.tsx +++ b/apps/rlark-ui/src/pages/Workflows.tsx @@ -25,6 +25,7 @@ import type { Copy } from "../i18n"; import type { DAGEdge, DAGNode, RoleResource, WorkflowJobDef } from "../types"; import type { CRDWorkflow } from "../crdModels"; import { useAutoRefresh } from "../hooks"; +import { copyText } from "../utils/clipboard"; import { crdToWorkflow } from "../utils/crd"; import { hasCycle, makeDefaultRoleResources } from "../utils/dag"; import { @@ -748,6 +749,7 @@ export function CreateWorkflowModal({ const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(""); const [workflowName, setWorkflowName] = useState("rl-training-pipeline"); + const [yamlCopied, setYamlCopied] = useState(false); useEffect(() => { const handleEscape = (event: KeyboardEvent) => { @@ -770,7 +772,7 @@ export function CreateWorkflowModal({ ); const [storageClasses, setStorageClasses] = useState< - Array<{ name: string; description: string; bucket: string }> + Array<{ id: string; name: string; description: string; bucket: string }> >([]); const [storageClassLoading, setStorageClassLoading] = useState(false); const [storageClassFetched, setStorageClassFetched] = useState(false); @@ -893,29 +895,27 @@ export function CreateWorkflowModal({ setStorageClassFetched(false); try { const data = await storageClassesApi.list<{ + storageClassName: string; name: string; description?: string; bucket?: string; - clusters?: string[]; - }>(); - const list = Object.values(data) - .filter((item) => !cluster || item.clusters?.includes(cluster)) - .map((item) => ({ - name: item.name, - description: item.description ?? "", - bucket: item.bucket ?? "", - })); + }>(cluster); + const list = Object.values(data).map((item) => ({ + id: item.storageClassName, + name: item.name, + description: item.description ?? "", + bucket: item.bucket ?? "", + })); setStorageClasses(list); } catch (e) { console.warn("Failed to fetch storage classes:", e); setStorageClasses( - mockStorageClasses - .filter((sc) => !cluster || sc.clusters.includes(cluster)) - .map(({ name, description, bucket }) => ({ - name, - description, - bucket, - })), + mockStorageClasses.map(({ id, name, description, bucket }) => ({ + id, + name, + description, + bucket, + })), ); } finally { setStorageClassLoading(false); @@ -1439,12 +1439,8 @@ export function CreateWorkflowModal({ }; return ( -
- e.target === e.currentTarget && !submitting && onClose() - } - > + // 表单内容无法恢复，遮罩点击不关闭弹层；关闭入口为右上角按钮、取消按钮和 Esc。 +
@@ -2092,7 +2088,7 @@ export function CreateWorkflowModal({ : "Select storage class"} {storageClasses.map((sc) => ( - ))} @@ -2243,9 +2239,14 @@ export function CreateWorkflowModal({ YAML 
{yaml}
diff --git a/apps/rlark-ui/src/styles/addons.css b/apps/rlark-ui/src/styles/addons.css index 664c236..a94306e 100644 --- a/apps/rlark-ui/src/styles/addons.css +++ b/apps/rlark-ui/src/styles/addons.css @@ -132,7 +132,8 @@ right: 0; bottom: 0; background: rgba(0, 0, 0, 0.35); - z-index: 1000; + /* 高于顶栏(1200)，否则抽屉遮罩盖不住顶栏 */ + z-index: 1240; } .addon-drawer { @@ -143,7 +144,7 @@ max-width: 90vw; height: 100vh; background: #fff; - z-index: 1001; + z-index: 1241; display: flex; flex-direction: column; box-shadow: -4px 0 24px rgba(0, 0, 0, 0.12); diff --git a/apps/rlark-ui/src/styles/clusters/management.css b/apps/rlark-ui/src/styles/clusters/management.css index a39e740..369401a 100644 --- a/apps/rlark-ui/src/styles/clusters/management.css +++ b/apps/rlark-ui/src/styles/clusters/management.css @@ -12,7 +12,17 @@ .cluster-management-table-panel { padding: 0; - overflow-x: auto; + overflow: visible; +} + +.cluster-management-table-body { + position: relative; + overflow: visible; +} + +.cluster-management-table-head, +.cluster-management-table-body { + border-radius: inherit; } .cluster-management-table-head, @@ -27,7 +37,7 @@ minmax(80px, 0.55fr) minmax(150px, 1fr) minmax(120px, 0.8fr) - 20px; + 36px; align-items: center; column-gap: 14px; } @@ -66,6 +76,61 @@ color: var(--soft); } +.cluster-actions { + position: relative; + z-index: 1; + display: inline-flex; + justify-content: flex-end; +} + +.cluster-actions .icon-button { + width: 32px; + height: 32px; + min-width: 32px; + border-radius: 50%; +} + +.cluster-actions:has(.cluster-action-menu) { + z-index: 11; +} + +.cluster-action-menu { + position: absolute; + z-index: 10; + top: calc(100% + 4px); + right: 0; + min-width: 150px; + display: grid; + padding: 5px; + border: 1px solid var(--line); + border-radius: 8px; + background: var(--panel); + box-shadow: 0 8px 22px rgba(15, 23, 42, 0.15); +} + +.cluster-action-menu button { + padding: 8px 10px; + border: 0; + border-radius: 6px; + background: transparent; + color: var(--ink); + text-align: left; + font-size: 12px; +} + +.cluster-action-menu button:hover:not(:disabled) { + background: var(--skyblue-bg); +} + +.cluster-action-menu button:disabled { + color: var(--soft); + cursor: not-allowed; +} + +.cluster-action-menu button.danger { + color: var(--red); +} + .cluster-management-name { min-width: 0; display: flex; diff --git a/apps/rlark-ui/src/styles/components/tag-and-filter.css b/apps/rlark-ui/src/styles/components/tag-and-filter.css index 8672384..9a1e965 100644 --- a/apps/rlark-ui/src/styles/components/tag-and-filter.css +++ b/apps/rlark-ui/src/styles/components/tag-and-filter.css @@ -237,10 +237,17 @@ color: var(--danger, #dc2626); background-color: #fef2f2; opacity: 1; - border: 1px solid var(--danger-border, #fecaca); + border: 1px solid #fecaca; border-radius: 6px; } +/* 深色主题下浅红底会在弹窗里形成刺眼色块，改为同色系的半透明底。 */ +.theme-dark .tag-editor-error { + color: #ff8a8a; + background-color: rgba(220, 38, 38, 0.16); + border-color: rgba(220, 38, 38, 0.42); +} + /* ============ TagFilterPopover ============ */ .tag-filter-popover { position: fixed; @@ -316,6 +323,13 @@ border-right: none; } +.tag-filter-hint { + margin: 8px 0 0; + font-size: 11px; + line-height: 1.5; + color: var(--muted); +} + .tag-filter-col-head { padding: 8px 12px; border-bottom: 1px solid var(--line); @@ -327,10 +341,23 @@ /* --bg 未定义会解析为透明，key 上翻时会与列头文字重叠，改用弹层背景色 --panel */ background: var(--panel); z-index: 1; + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + min-height: 31px; } .tag-filter-col-title { font-weight: 600; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.tag-filter-col-head .tag-filter-link-btn { + flex: none; } .tag-filter-link-btn { @@ -364,12 +391,39 @@ } .tag-filter-key-item.selected { - background: var(--blue-bg, #eef2ff); + background: var(--blue-bg); +} + +/* 当前在右栏展示值的 key：用右侧描边和强调色标出，与「已勾选」区分开。 */ +.tag-filter-key-item.active { border-right-color: var(--blue); } -.tag-filter-key-item.hovered { - background: var(--hover, #f3f4f6); +.tag-filter-key-item.active .tag-filter-key-name { + color: var(--blue); + font-weight: 600; +} + +/* key 行左侧是勾选框，右侧整块是切换右栏的按钮，两者互不干扰。 */ +.tag-filter-key-main { + display: flex; + flex: 1; + align-items: center; + gap: 8px; + min-width: 0; + padding: 0; + border: none; + background: transparent; + color: inherit; + font: inherit; + text-align: left; + cursor: pointer; +} + +.tag-filter-key-name { + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; } .tag-filter-key-item input, @@ -507,7 +561,7 @@ } .col-filter-item.selected { - background: var(--blue-bg, #eef2ff); + background: var(--blue-bg); } .col-filter-item input { diff --git a/apps/rlark-ui/src/styles/foundation/shell.css b/apps/rlark-ui/src/styles/foundation/shell.css index 1d50be9..b6dbc5e 100644 --- a/apps/rlark-ui/src/styles/foundation/shell.css +++ b/apps/rlark-ui/src/styles/foundation/shell.css @@ -34,6 +34,7 @@ --shadow-soft: 0 8px 24px rgba(28, 39, 58, 0.055); --app-bg: #f4f6f9; --hover: #f5f0fe; + --blue-bg: #efe9fd; } * { @@ -558,6 +559,42 @@ button { border-radius: 10px; } +/* 复制成功反馈：图标切换为对勾的同时给出一次短暂的高亮脉冲， + 让用户在不移动视线的情况下确认复制已经完成。 */ +@keyframes icon-button-copy-pulse { + 0% { + transform: scale(1); + box-shadow: 0 0 0 0 rgba(50, 201, 141, 0.45); + } + 45% { + transform: scale(1.12); + box-shadow: 0 0 0 6px rgba(50, 201, 141, 0); + } + 100% { + transform: scale(1); + box-shadow: 0 0 0 0 rgba(50, 201, 141, 0); + } +} + +.icon-button.copy-confirmed { + border-color: #32c98d; + color: #1f9d6d; + background: #e8f9f1; + animation: icon-button-copy-pulse 0.42s ease-out; +} + +.theme-dark .icon-button.copy-confirmed { + border-color: #2fa877; + color: #5fe0ac; + background: #11291f; +} + +@media (prefers-reduced-motion: reduce) { + .icon-button.copy-confirmed { + animation: none; + } +} + .primary-button { height: 40px; padding: 0 14px; @@ -1285,7 +1322,10 @@ button { opacity: 0; pointer-events: none; transition: opacity 0.12s; - z-index: 999; + /* 该 tooltip 通过 portal 挂到 app shell 并用 fixed 定位，必须叠在 sticky 的 + .topbar 之上，否则靠近页面顶部的徽章提示会被顶栏遮挡；同时保持在 + 弹窗遮罩之下。 */ + z-index: 1260; display: flex; flex-direction: column; gap: 6px; @@ -1418,6 +1458,14 @@ button { color: #2f8a60; } +/* 已被后续进展取代的历史事件（如绑定节点前的 FailedScheduling）：保留可 + 排查性，但用中性灰 + 删除线弱化，避免被误读为当前阻塞原因。 */ +.status-info-tooltip .event-entry .event-chip.event-resolved { + background: #f1f3f6; + color: #8b95a5; + text-decoration: line-through; +} + .theme-dark .status-info { border-color: #344156; background: #1b2637; diff --git a/apps/rlark-ui/src/styles/jobs/code-editor-and-list.css b/apps/rlark-ui/src/styles/jobs/code-editor-and-list.css index f662032..5b9e195 100644 --- a/apps/rlark-ui/src/styles/jobs/code-editor-and-list.css +++ b/apps/rlark-ui/src/styles/jobs/code-editor-and-list.css @@ -220,7 +220,10 @@ color: #27303d; outline: 0; font-size: 12px; - background: #fff; + /* 必须用 background-color 而非 background 简写：简写会把全局 select 规则提供的 + background-image（下拉箭头）重置为 none，而 appearance: none 已移除原生箭头， + 结果是控件完全没有下拉指示。 */ + background-color: #fff; cursor: pointer; } @@ -235,7 +238,13 @@ background: #fff; } -.theme-dark .device-row select, +/* select 单独成规则并只覆盖 background-color，原因同 .device-row select。 */ +.theme-dark .device-row select { + background-color: #151d2b; + color: #c4cbd6; + border-color: rgba(255, 255, 255, 0.08); +} + .theme-dark .device-row input { background: #151d2b; color: #c4cbd6; @@ -328,7 +337,8 @@ color: #27303d; outline: 0; font-size: 12px; - background: #fff; + /* 同 .device-row select：用 background 简写会抹掉全局 select 的下拉箭头。 */ + background-color: #fff; cursor: pointer; } @@ -349,12 +359,26 @@ height: 100%; border: 0; border-radius: 0; +} + +.mount-row .mount-field-box input { background: transparent; } +/* 同 .device-row select：用 background 简写会抹掉全局 select 的下拉箭头。 */ +.mount-row .mount-field-box select { + background-color: transparent; +} + +/* select 单独成规则并只覆盖 background-color，原因同 .device-row select。 */ .theme-dark .mount-row select, -.theme-dark .mount-row input, .theme-dark .env-row select { + background-color: #151d2b; + border-color: var(--line); + color: #d8e0ee; +} + +.theme-dark .mount-row input { background: #151d2b; border-color: var(--line); color: #d8e0ee; @@ -371,11 +395,15 @@ color: #9ca8ba; } -.theme-dark .mount-row .mount-field-box select, .theme-dark .mount-row .mount-field-box input { background: transparent; } +/* select 单独成规则并只覆盖 background-color，原因同 .device-row select。 */ +.theme-dark .mount-row .mount-field-box select { + background-color: transparent; +} + .env-row input { width: 100%; border: 1px solid var(--line); @@ -784,11 +812,17 @@ color: #f5f7fb; } +/* select 单独成规则并只覆盖 background-color：用 background 简写会连带重置 + 全局 select 规则提供的下拉箭头（background-image/repeat/position/size）。 */ +.theme-dark .metrics-filter-bar select, +.theme-dark .worker-filter-bar select { + background-color: #151d2b; + border-color: var(--line); +} + .theme-dark .sub-tabs, .theme-dark .metrics-scope-toggle, .theme-dark .worker-filter-bar, -.theme-dark .metrics-filter-bar select, -.theme-dark .worker-filter-bar select, .theme-dark .time-series-card, .theme-dark .public-config-card, .theme-dark .task-summary-metric, diff --git a/apps/rlark-ui/src/styles/jobs/create.css b/apps/rlark-ui/src/styles/jobs/create.css index f13bfb1..94e9541 100644 --- a/apps/rlark-ui/src/styles/jobs/create.css +++ b/apps/rlark-ui/src/styles/jobs/create.css @@ -237,6 +237,25 @@ color: #f5f7fb; } +/* 浅色模式聚焦时用白底提升对比度；深色模式必须改用深色底， + 否则白底叠加浅色文字会让正在输入的角色名完全不可见。 */ +.theme-dark .role-edit-row input:focus { + background: #0f1725; + border-color: #a78bfa; +} + +.theme-dark .role-edit-row input::placeholder { + color: #7c8798; +} + +.theme-dark .role-edit-row small { + color: #9ca8ba; +} + +.theme-dark .role-edit-row.active small { + color: #c4b5fd; +} + .role-resource-grid { display: grid; grid-template-columns: 1fr; diff --git a/apps/rlark-ui/src/styles/jobs/detail-foundation.css b/apps/rlark-ui/src/styles/jobs/detail-foundation.css index 586eb62..2b742b1 100644 --- a/apps/rlark-ui/src/styles/jobs/detail-foundation.css +++ b/apps/rlark-ui/src/styles/jobs/detail-foundation.css @@ -570,6 +570,25 @@ font-size: 13px; } +.pod-subtable-truncated { + display: flex; + align-items: center; + gap: 4px; + min-width: 0; + max-width: 260px; +} + +.pod-subtable-truncated > svg { + flex: none; +} + +.pod-subtable-truncated > span { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + .pod-subtable tbody tr:nth-child(even) td { background: #f6f9fd; } @@ -1448,6 +1467,12 @@ font-size: 10px; } +.job-ssh-key-list strong.job-ssh-key-owner-removed { + color: var(--muted); + font-style: italic; + font-weight: 500; +} + .public-basic-config-list .job-ssh-key-list code { padding: 2px 0; border: 0; @@ -1598,7 +1623,7 @@ .job-detail-page .role-runtime-config.job-detail-summary-card { border: 1px solid var(--line); border-radius: 18px; - background: #fff; + background: var(--panel); box-shadow: var(--shadow-soft); padding: 16px; position: relative; @@ -1834,18 +1859,6 @@ font-weight: 850; } -.restart-choice-backdrop { - z-index: 1200; -} - -.delete-job-backdrop { - z-index: 1200; -} - -.job-lifecycle-backdrop { - z-index: 1200; -} - .job-lifecycle-modal { width: min(500px, calc(100vw - 32px)); border-radius: 18px; @@ -1860,6 +1873,10 @@ border-bottom: 1px solid var(--line); } +.cluster-action-confirm-head { + grid-template-columns: minmax(0, 1fr) 34px; +} + .job-lifecycle-icon { display: grid; place-items: center; diff --git a/apps/rlark-ui/src/styles/jobs/detail-runtime.css b/apps/rlark-ui/src/styles/jobs/detail-runtime.css index 712451b..2280f33 100644 --- a/apps/rlark-ui/src/styles/jobs/detail-runtime.css +++ b/apps/rlark-ui/src/styles/jobs/detail-runtime.css @@ -4,7 +4,7 @@ gap: 8px; padding: 12px 18px; border: 0; - background: #fff; + background: var(--panel); } .worker-filter-bar label, @@ -24,8 +24,10 @@ height: 34px; border: 1px solid var(--line); border-radius: 9px; - background: #fff; - color: var(--text); + /* 必须用 background-color 而非 background 简写：简写会连带重置全局 select 规则 + 提供的 background-image/repeat/position/size，导致下拉箭头平铺铺满控件。 */ + background-color: var(--panel); + color: var(--ink); padding: 0 28px 0 9px; font-size: 12px; outline: 0; @@ -136,8 +138,8 @@ margin-bottom: 14px; border: 1px solid color-mix(in srgb, #23a36d 35%, var(--line)); border-radius: 10px; - background: color-mix(in srgb, #23a36d 8%, var(--surface)); - color: var(--text); + background: color-mix(in srgb, #23a36d 8%, var(--panel)); + color: var(--ink); padding: 10px 12px; font-size: 13px; font-weight: 650; @@ -281,7 +283,7 @@ .worker-detail-head strong { display: block; margin-top: 4px; - color: var(--text); + color: var(--ink); font-size: 13px; } @@ -305,7 +307,7 @@ min-width: 0; border: 1px solid var(--line); border-radius: 10px; - background: #fff; + background: var(--panel); overflow: hidden; } @@ -350,7 +352,7 @@ margin: 0; padding: 9px 10px; border-bottom: 1px solid var(--line); - background: #fff; + background: var(--panel); } .role-runtime-selector-card p:last-child { @@ -362,11 +364,11 @@ display: block; width: auto; max-width: 100%; - border: 1px solid #e6ebf2; + border: 1px solid var(--line); border-radius: 7px; - background: #f8fafc; + background: var(--canvas); padding: 5px 7px; - color: #475569; + color: var(--ink); font-family: "JetBrains Mono", monospace; font-size: 10px; overflow-wrap: anywhere; @@ -378,8 +380,8 @@ width: 20px; height: 20px; border-radius: 999px; - background: #f1f4f8; - color: #7b8797; + background: var(--canvas); + color: var(--muted); font-size: 11px; } @@ -394,7 +396,7 @@ min-width: 0; border: 1px solid var(--line); border-radius: 10px; - background: #fff; + background: var(--panel); overflow: hidden; } @@ -427,16 +429,16 @@ .role-runtime-table-card th { color: var(--muted); - background: #eef3f9; + background: var(--canvas); font-weight: 800; } .role-runtime-table-card td { - background: #fbfdff; + background: var(--panel); } .role-runtime-table-card tbody tr:nth-child(even) td { - background: #f6f9fd; + background: var(--canvas); } .role-runtime-table-card tr:last-child td { @@ -447,9 +449,9 @@ display: inline-flex; max-width: 100%; border-radius: 6px; - background: #eef3fb; + background: var(--canvas); padding: 4px 7px; - color: #39475a; + color: var(--ink); font-family: "JetBrains Mono", monospace; font-size: 10px; white-space: pre-wrap; @@ -596,7 +598,7 @@ flex-wrap: wrap; margin: 16px 18px; padding: 12px; - background: var(--bg-secondary, #f8f9fa); + background: var(--canvas); border-radius: 8px; border: 1px solid var(--line); } @@ -610,9 +612,9 @@ height: 36px; border: 1px solid var(--line); border-radius: 6px; - background: #fff; + background: var(--panel); padding: 0 12px; - color: #8793a6; + color: var(--muted); } .log-search-field input { @@ -621,7 +623,7 @@ border: 0; outline: 0; background: transparent; - color: var(--text); + color: var(--ink); font-size: 13px; } @@ -632,16 +634,19 @@ height: 36px; border: 1px solid var(--line); border-radius: 6px; - background: #fff; + background: var(--panel); padding: 0 12px; - color: #8793a6; + color: var(--muted); } .log-order-filter select { border: 0; outline: none; - background: transparent; - color: var(--text); + /* 该控件由 .log-order-filter::after 自绘箭头，需显式关闭全局 select 的背景箭头， + 同时避免用 background 简写（简写会把 repeat/position/size 重置成平铺）。 */ + background-color: transparent; + background-image: none; + color: var(--ink); font-size: 13px; cursor: pointer; -webkit-appearance: none; @@ -655,6 +660,13 @@ box-shadow: none; } +/* .theme-dark select 的特异性（0,1,1）与 .log-order-filter select 相同且导入更靠后， + 会把背景箭头重新画回来，导致深色下出现 SVG 箭头 + ::after "▼" 的双箭头。 */ +.theme-dark .log-order-filter select { + background-color: transparent; + background-image: none; +} + .log-order-filter::after { content: "▼"; font-size: 10px; @@ -667,16 +679,16 @@ height: 36px; border: 1px solid var(--line); border-radius: 6px; - background: #fff; + background: var(--panel); padding: 0 12px; font-size: 13px; - color: var(--text); + color: var(--ink); font-family: inherit; } .log-custom-datetime:focus { outline: none; - border-color: var(--accent); + border-color: var(--blue); box-shadow: 0 0 0 2px rgba(99, 102, 241, 0.1); } @@ -687,8 +699,8 @@ height: 36px; border: 1px solid var(--line); border-radius: 6px; - background: #fff; - color: #788598; + background: var(--panel); + color: var(--muted); padding: 0 12px; font-size: 12px; font-weight: 600; @@ -743,8 +755,8 @@ margin: 0; border: none; border-radius: 0; - /* 高于侧栏(5)、顶栏(4)、tooltip(999)，低于弹窗(1200) */ - z-index: 1100; + /* 高于侧栏(5)、tooltip(999)、顶栏(1200)，低于弹窗遮罩(1300) */ + z-index: 1250; /* 不透明深色，彻底盖住下层内容 */ background: #0d1520; display: flex; @@ -885,7 +897,7 @@ font-size: 10px; } .log-worker-name { - color: var(--text); + color: var(--ink); font-family: "JetBrains Mono", monospace; font-size: 11px; font-weight: 750; @@ -966,7 +978,7 @@ } .metrics-integration-state strong { - color: var(--text); + color: var(--ink); font-size: 16px; } @@ -1000,14 +1012,14 @@ height: 32px; border: 1px solid var(--line); border-radius: 8px; - background: #fff; - color: var(--text); + background-color: var(--panel); + color: var(--ink); padding: 0 24px 0 9px; font-size: 11px; } .metrics-filter-bar select:disabled { - background: #f3f5f8; - color: #9aa4b2; + background-color: var(--canvas); + color: var(--muted); cursor: not-allowed; } .metrics-scope-toggle { @@ -1080,7 +1092,7 @@ .time-series-head strong { display: block; margin-top: 5px; - color: var(--text); + color: var(--ink); font-size: 20px; } .time-series-head strong small { @@ -1157,7 +1169,7 @@ .role-worker-group-head > div > strong, .role-worker-resource-summary strong { display: block; - color: var(--text); + color: var(--ink); font-size: 14px; line-height: 1.45; } @@ -1205,7 +1217,7 @@ .worker-access-head strong { display: block; - color: var(--text); + color: var(--ink); font-size: 13px; line-height: 1.35; overflow: hidden; @@ -1257,7 +1269,7 @@ .worker-access-meta b { display: block; margin-top: 2px; - color: var(--text); + color: var(--ink); font-size: 11px; overflow: hidden; text-overflow: ellipsis; @@ -1316,7 +1328,7 @@ .config-section-head h3 { margin: 4px 0 0; - color: var(--text); + color: var(--ink); font-size: 16px; line-height: 1.35; } @@ -1358,7 +1370,7 @@ .config-kv-card strong { margin-top: 6px; - color: var(--text); + color: var(--ink); font-size: 14px; } @@ -1474,7 +1486,7 @@ } .role-config-head strong { - color: var(--text); + color: var(--ink); font-size: 12px; } @@ -1498,7 +1510,7 @@ .role-config-facts dd { min-width: 0; margin: 0; - color: var(--text); + color: var(--ink); font-size: 11px; line-height: 1.45; overflow-wrap: anywhere; diff --git a/apps/rlark-ui/src/styles/jobs/tags-and-editing.css b/apps/rlark-ui/src/styles/jobs/tags-and-editing.css index 6a43ed7..4589bb2 100644 --- a/apps/rlark-ui/src/styles/jobs/tags-and-editing.css +++ b/apps/rlark-ui/src/styles/jobs/tags-and-editing.css @@ -32,9 +32,10 @@ color: var(--blue); } +/* 只收窄左右内边距；垂直方向沿用表格默认的 middle， + 否则标签会比同行的类型 / 状态列高出一截。 */ .job-table-tag-cell { padding: 10px 12px; - vertical-align: top; } .job-tags-cell { @@ -53,7 +54,7 @@ font-weight: 500; background: transparent; color: var(--muted); - border: 1px solid #d1d5db; + border: 1px solid var(--line-strong); border-radius: 3px; line-height: 1.5; white-space: nowrap; @@ -72,8 +73,8 @@ max-width: min(360px, calc(100vw - 24px)); max-height: calc(100vh - 24px); padding: 10px; - background: var(--card-bg, #fff); - border: 1px solid #d1d5db; + background: var(--panel); + border: 1px solid var(--line-strong); border-radius: 6px; box-shadow: 0 8px 20px rgb(0 0 0 / 12%); } @@ -304,13 +305,19 @@ padding: 3px 10px; font-size: 12px; font-weight: 500; - background: var(--blue-bg, #eef2ff); - color: var(--blue, #6366f1); + background: #eef2ff; + color: var(--blue); border: 1px solid rgba(99, 102, 241, 0.25); border-radius: 5px; line-height: 1.4; } +/* 深色主题下浅紫底会在面板上形成高亮色块，改为同色系的半透明底。 */ +.theme-dark .job-tag-chip-inline { + background: rgba(99, 102, 241, 0.18); + border-color: rgba(99, 102, 241, 0.4); +} + .public-tags-editor-actions { display: flex; align-items: center; diff --git a/apps/rlark-ui/src/styles/nodes/files-and-selector.css b/apps/rlark-ui/src/styles/nodes/files-and-selector.css index b646a59..939f21f 100644 --- a/apps/rlark-ui/src/styles/nodes/files-and-selector.css +++ b/apps/rlark-ui/src/styles/nodes/files-and-selector.css @@ -183,6 +183,10 @@ tr.clickable:hover { color: var(--muted, #8b95a7); } +.field-hint.field-hint-warning { + color: var(--danger, #e05270); +} + .label-row { display: flex; align-items: baseline; diff --git a/apps/rlark-ui/src/styles/nodes/insight.css b/apps/rlark-ui/src/styles/nodes/insight.css index 6eefa57..72b659e 100644 --- a/apps/rlark-ui/src/styles/nodes/insight.css +++ b/apps/rlark-ui/src/styles/nodes/insight.css @@ -520,7 +520,7 @@ .node-worker-table-wrap .node-worker-table { width: 100%; - min-width: 1280px; + min-width: 860px; border-collapse: collapse; } @@ -562,6 +562,15 @@ text-overflow: ellipsis; } +.node-worker-name { + display: block; + min-width: 0; + max-width: 200px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + .node-worker-empty { padding: 24px; border: 1px dashed var(--line); diff --git a/apps/rlark-ui/src/styles/overrides/global-responsive.css b/apps/rlark-ui/src/styles/overrides/global-responsive.css index f54e9dd..806a165 100644 --- a/apps/rlark-ui/src/styles/overrides/global-responsive.css +++ b/apps/rlark-ui/src/styles/overrides/global-responsive.css @@ -11,7 +11,7 @@ minmax(48px, 0.42fr) minmax(104px, 0.8fr) minmax(82px, 0.65fr) - 16px; + 36px; column-gap: 8px; } diff --git a/apps/rlark-ui/src/styles/shared/content.css b/apps/rlark-ui/src/styles/shared/content.css index a0cc8dc..341e505 100644 --- a/apps/rlark-ui/src/styles/shared/content.css +++ b/apps/rlark-ui/src/styles/shared/content.css @@ -81,7 +81,8 @@ flex: 1; border: 0; outline: 0; - background: transparent; + /* 用 background-color 而非 background 简写，避免重置全局 select 的下拉箭头。 */ + background-color: transparent; color: var(--ink); font-size: 12px; font-weight: 700; @@ -282,7 +283,7 @@ .job-tab-panel { border: 1px solid var(--line); border-radius: 18px; - background: #fff; + background: var(--panel); box-shadow: var(--shadow-soft); overflow: hidden; } @@ -327,10 +328,10 @@ content: ""; display: block; height: 6px; - background: #f7f8fb; + background: var(--canvas); margin: 0 -16px; - border-top: 1px solid #eef0f4; - border-bottom: 1px solid #eef0f4; + border-top: 1px solid var(--line); + border-bottom: 1px solid var(--line); } .dag-canvas { @@ -1624,8 +1625,15 @@ tbody tr:hover { .modal-backdrop { position: fixed; inset: 0; - /* 必须高于 .action-dropdown(50),否则打开弹窗时下拉菜单会浮在遮罩之上 */ - z-index: 60; + /* + * 全屏遮罩层级基线。顶栏 .topbar 是 sticky 元素,与遮罩同处根层叠上下文, + * 遮罩层级低于顶栏时顶栏不会被遮住,因此全屏遮罩必须排在顶栏之上。 + * 升序约定: + * .sidebar(5) < .action-dropdown(50) < .status-info-tooltip(999) + * < .topbar(1200) < .addon-drawer-overlay(1240)/.addon-drawer(1241) + * < .log-terminal-fullscreen(1250) < .modal-backdrop(1300) + */ + z-index: 1300; display: grid; place-items: center; background: rgba(16, 22, 32, 0.46); @@ -1725,9 +1733,20 @@ tbody tr:hover { color: #27303d; outline: 0; font-size: 12px; +} + +.modal-body input, +.modal-body textarea { background: #fff; } +/* select 只覆盖 background-color：用 background 简写会把全局 select 规则提供的 + background-image（下拉箭头）重置为 none，而 appearance: none 已移除原生箭头， + 结果是控件完全没有下拉指示，右侧只剩一块空白内边距。 */ +.modal-body select { + background-color: #fff; +} + .modal-body input:focus, .modal-body textarea:focus, .modal-body select:focus { @@ -1897,6 +1916,7 @@ tbody tr:hover { --shadow-soft: 0 10px 30px rgba(0, 0, 0, 0.22); --app-bg: #0c111b; --hover: rgba(167, 139, 250, 0.08); + --blue-bg: rgba(167, 139, 250, 0.18); color: #f5f7fb; background: radial-gradient( @@ -2110,13 +2130,19 @@ tbody tr:hover { .theme-dark .live-chip, .theme-dark .map-live, .theme-dark .modal-body input, -.theme-dark .modal-body textarea, -.theme-dark .modal-body select { +.theme-dark .modal-body textarea { background: #111a29; border-color: var(--line); color: #d8e0ee; } +/* 原因同 .modal-body select：select 单独成规则并只覆盖 background-color。 */ +.theme-dark .modal-body select { + background-color: #111a29; + border-color: var(--line); + color: #d8e0ee; +} + .theme-dark .topbar > div:first-child:has(h1)::before { border-color: #687895; } @@ -2128,13 +2154,19 @@ tbody tr:hover { background: #101827; } -.theme-dark .toolbar-filter, -.theme-dark .toolbar-filter select { +.theme-dark .toolbar-filter { background: #151d2b; color: #d8e0ee; border-color: var(--line); } +/* 原因同 .toolbar-filter select。 */ +.theme-dark .toolbar-filter select { + background-color: #151d2b; + color: #d8e0ee; + border-color: var(--line); +} + .theme-dark .metric-card::after { background: rgba(167, 139, 250, 0.08); } diff --git a/apps/rlark-ui/src/styles/shared/table-actions-and-controls.css b/apps/rlark-ui/src/styles/shared/table-actions-and-controls.css index ab40ba3..d86d684 100644 --- a/apps/rlark-ui/src/styles/shared/table-actions-and-controls.css +++ b/apps/rlark-ui/src/styles/shared/table-actions-and-controls.css @@ -124,7 +124,7 @@ padding: 0 10px; border: 1px solid var(--line); border-radius: 9px; - background: var(--surface, #fff); + background: var(--panel); color: var(--blue); font-size: 12px; font-weight: 700; @@ -171,8 +171,8 @@ padding: 0 9px; border: 1px solid var(--line); border-radius: 8px; - background: var(--surface); - color: var(--text); + background: var(--panel); + color: var(--ink); font-size: 12px; font-weight: 600; cursor: pointer; @@ -230,8 +230,8 @@ min-width: 140px; padding: 4px; border-radius: 10px; - background: var(--surface, #fff); - border: 1px solid var(--border, #e5e5ec); + background: var(--panel); + border: 1px solid var(--line-strong); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.08); display: flex; flex-direction: column; @@ -247,13 +247,13 @@ background: transparent; border-radius: 6px; font-size: 13px; - color: var(--text, #1a1a2e); + color: var(--ink); cursor: pointer; transition: background 0.12s; } .action-dropdown-item:hover:not(:disabled) { - background: var(--hover, #f5f5fa); + background: var(--hover); } .action-dropdown-item:disabled { @@ -262,7 +262,15 @@ } .action-dropdown-item.danger { - color: var(--red, #cf3f61); + color: var(--red); +} + +.action-dropdown-item.danger:hover:not(:disabled) { + background: rgba(239, 90, 122, 0.08); +} + +.theme-dark .action-dropdown-item.danger:hover:not(:disabled) { + background: rgba(239, 90, 122, 0.16); } .clickable-row { @@ -270,7 +278,7 @@ } .clickable-row:hover { - background: var(--surface-hover, rgba(0, 0, 0, 0.03)); + background: var(--hover); } .pagination-bar { diff --git a/apps/rlark-ui/src/styles/storage/create-dialog.css b/apps/rlark-ui/src/styles/storage/create-dialog.css index 30d0f89..10fc204 100644 --- a/apps/rlark-ui/src/styles/storage/create-dialog.css +++ b/apps/rlark-ui/src/styles/storage/create-dialog.css @@ -89,7 +89,6 @@ width: 100%; border: 1px solid var(--line-strong); border-radius: 10px; - background: var(--panel); color: var(--ink); outline: none; transition: @@ -97,6 +96,17 @@ box-shadow 0.15s; } +.storage-create-form input:not([type="checkbox"]), +.storage-create-form textarea { + background: var(--panel); +} + +/* select 只覆盖 background-color：用 background 简写会把全局 select 规则提供的 + 下拉箭头（background-image）重置为 none。 */ +.storage-create-form select { + background-color: var(--panel); +} + .storage-create-form input:not([type="checkbox"]), .storage-create-form select { height: 42px; @@ -340,11 +350,63 @@ accent-color: var(--blue); } +.registry-cluster-select { + position: relative; + margin-top: 12px; +} + +.storage-create-form .registry-cluster-trigger { + width: 100%; + min-height: 42px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 9px 11px; + border: 1px solid var(--line-strong); + border-radius: 10px; + background: var(--canvas); + color: var(--ink); + cursor: pointer; + font: inherit; + font-size: 13px; + text-align: left; +} + +.storage-create-form .registry-cluster-trigger:hover, +.storage-create-form .registry-cluster-trigger:focus-visible { + border-color: var(--blue); + box-shadow: 0 0 0 3px rgba(124, 58, 237, 0.12); + outline: none; +} + +.registry-cluster-trigger svg { + flex: 0 0 auto; + color: var(--muted); + transform: rotate(90deg); + transition: transform 0.15s ease; +} + +.registry-cluster-trigger svg.open { + transform: rotate(-90deg); +} + .registry-cluster-options { + position: absolute; + z-index: 3; + top: calc(100% + 6px); + right: 0; + left: 0; + max-height: 240px; display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; - margin-top: 12px; + overflow-y: auto; + padding: 8px; + border: 1px solid var(--line-strong); + border-radius: 10px; + background: var(--panel); + box-shadow: 0 12px 28px rgba(15, 23, 42, 0.16); } .storage-create-form .registry-cluster-options label { @@ -405,14 +467,50 @@ padding: 10px 0; } -.storage-create-form .error-text { - margin: 0; - padding: 10px 12px; - border: 1px solid rgba(239, 90, 122, 0.25); +.storage-create-error { + position: absolute; + z-index: 4; + top: 24px; + left: 50%; + transform: translateX(-50%); + width: min(460px, calc(100% - 52px)); + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: start; + gap: 10px; + padding: 12px; + border: 1px solid rgba(239, 90, 122, 0.35); border-radius: 10px; - background: rgba(239, 90, 122, 0.08); + background: var(--panel); + box-shadow: 0 12px 30px rgba(13, 22, 38, 0.2); color: var(--red); font-size: 12px; + line-height: 1.5; +} + +.storage-create-error svg { + margin-top: 1px; + flex: none; +} + +.storage-create-error-close { + width: 24px; + height: 24px; + display: grid; + place-items: center; + margin: -3px -3px -3px 0; + padding: 0; + border: 0; + border-radius: 6px; + background: transparent; + color: inherit; + cursor: pointer; +} + +.storage-create-error-close:hover, +.storage-create-error-close:focus-visible { + background: rgba(239, 90, 122, 0.12); + outline: 0; } .storage-create-form .form-actions { @@ -435,13 +533,19 @@ .theme-dark .storage-create-form .form-section, .theme-dark .storage-create-form input:not([type="checkbox"]), -.theme-dark .storage-create-form select, .theme-dark .storage-create-form textarea { background: #151d2b; border-color: #33425b; color: #e6edf7; } +/* 原因同 .storage-create-form select。 */ +.theme-dark .storage-create-form select { + background-color: #151d2b; + border-color: #33425b; + color: #e6edf7; +} + .theme-dark .storage-create-form .form-grid @@ -500,6 +604,12 @@ margin-top: 6px; } + .storage-create-error { + right: 16px; + bottom: 72px; + max-width: calc(100% - 32px); + } + .storage-create-form .form-actions { margin-left: -16px; margin-right: -16px; diff --git a/apps/rlark-ui/src/types.ts b/apps/rlark-ui/src/types.ts index 071a246..6913f96 100644 --- a/apps/rlark-ui/src/types.ts +++ b/apps/rlark-ui/src/types.ts @@ -1,5 +1,6 @@ import type { Activity, LayoutDashboard } from "lucide-react"; -import type { JobType, PullProgressEntry } from "./data"; +import type { JobType, MountType, PullProgressEntry } from "./data"; +import type { CRDVolume } from "./crdModels"; export type Page = | "overview" @@ -28,6 +29,7 @@ export interface ResourceRow { export interface ClusterSummary { id: string; + agentNamespace: string; name: string; type: "Cloud" | "Embodied" | "Hybrid" | string; region: string; @@ -36,9 +38,9 @@ export interface ClusterSummary { | "Online" | "Offline" | "Downgraded" + | "Delisted" | "Cordoned" | "Draining" - | "Deleted" | string; totalNodes: number; onlineNodes: number; @@ -50,6 +52,7 @@ export interface ClusterSummary { robotModels: string[]; runningJobs: number; description: string; + agentNamespaceConflict?: boolean; } export interface CRDPod { @@ -109,11 +112,12 @@ export interface RoleResource { prepareScript: string; envs: Array<{ key: string; value: string }>; mounts: Array<{ - type: "host" | "storage"; + type: MountType; objectStorage: string; mountPath: string; hostPath: string; pvcSizeGb: number | ""; + rawVolume?: CRDVolume; }>; } diff --git a/apps/rlark-ui/src/utils/clipboard.ts b/apps/rlark-ui/src/utils/clipboard.ts new file mode 100644 index 0000000..c7f368e --- /dev/null +++ b/apps/rlark-ui/src/utils/clipboard.ts @@ -0,0 +1,30 @@ +// copyText 把文本写入系统剪贴板，并返回是否复制成功。 +// +// 浏览器只在安全上下文（HTTPS 或 localhost）下暴露 navigator.clipboard， +// 控制台经 HTTP 访问时该对象为 undefined，直接调用会抛 TypeError。 +// 因此这里先尝试异步 Clipboard API，失败或不可用时回退到 +// document.execCommand("copy")，保证两种部署方式下复制都可用。 +export async function copyText(value: string): Promise { + if (!value) return false; + + try { + if (navigator.clipboard?.writeText) { + await navigator.clipboard.writeText(value); + return true; + } + } catch { + // Fall through for browsers that block Clipboard API on local HTTP pages. + } + + const textarea = document.createElement("textarea"); + textarea.value = value; + textarea.style.position = "fixed"; + textarea.style.opacity = "0"; + document.body.appendChild(textarea); + textarea.focus({ preventScroll: true }); + textarea.select(); + textarea.setSelectionRange(0, value.length); + const copied = document.execCommand("copy"); + textarea.remove(); + return copied; +} diff --git a/apps/rlark-ui/src/utils/crd.ts b/apps/rlark-ui/src/utils/crd.ts index 00f969e..7269b6e 100644 --- a/apps/rlark-ui/src/utils/crd.ts +++ b/apps/rlark-ui/src/utils/crd.ts @@ -1,5 +1,5 @@ -import type { Job, JobTag, JobType, Phase } from "../data"; -import type { CRDJob, CRDJobTask, CRDWorkflow } from "../crdModels"; +import type { Job, JobTag, JobType, MountType, Phase } from "../data"; +import type { CRDJob, CRDJobTask, CRDVolume, CRDWorkflow } from "../crdModels"; // 生成稳定的 tag id：用 key:value 组合哈希，保证同一 CRD 多次转换得到相同 id。 function tagId(key: string, value: string): string { @@ -19,6 +19,100 @@ function storageGi(quantity?: string | number): number { return match ? Number(match[1]) : 10; } +type JobMount = { + type: MountType; + objectStorage: string; + mountPath: string; + hostPath: string; + pvcSizeGb: number; + rawVolume?: CRDVolume; +}; + +// 把 Pod 模板里的一个 volume 映射成表单可消费的挂载项。 +// 表单只能编辑 hostPath 与 ephemeral/PVC 两类卷；其余卷源（emptyDir、configMap、 +// secret、projected 等）标记为 opaque 并原样保留 rawVolume，使"复制任务"不会把它们 +// 降级成空的主机目录——例如 /dev/shm 的 emptyDir{medium:Memory} 必须保持内存 tmpfs。 +export function toJobMount( + vol: CRDVolume | undefined, + mountPath: string, +): JobMount { + if (vol?.ephemeral) { + const spec = vol.ephemeral.volumeClaimTemplate?.spec; + return { + type: "storage", + objectStorage: spec?.storageClassName ?? "", + mountPath, + hostPath: "", + pvcSizeGb: storageGi(spec?.resources?.requests?.storage), + }; + } + if (vol?.persistentVolumeClaim) { + return { + type: "storage", + objectStorage: vol.persistentVolumeClaim.claimName, + mountPath, + hostPath: "", + pvcSizeGb: 10, + }; + } + if (vol?.hostPath) { + return { + type: "host", + objectStorage: "", + mountPath, + hostPath: vol.hostPath.path, + pvcSizeGb: 10, + }; + } + if (vol) { + return { + type: "opaque", + objectStorage: "", + mountPath, + hostPath: "", + pvcSizeGb: 10, + rawVolume: vol, + }; + } + // volumeMount 找不到对应的 volume：按空主机目录展示，提示用户补全。 + return { + type: "host", + objectStorage: "", + mountPath, + hostPath: "", + pvcSizeGb: 10, + }; +} + +// 把表单无法编辑的卷源摘要成一行只读文案，例如 "emptyDir (Memory, 32Gi)"。 +export function describeVolumeSource(vol?: CRDVolume): string { + if (!vol) return ""; + const entry = Object.entries(vol).find(([key]) => key !== "name"); + if (!entry) return ""; + const [kind, source] = entry; + if (kind === "emptyDir") { + const empty = source as { medium?: string; sizeLimit?: string | number }; + const detail = [empty?.medium, empty?.sizeLimit].filter(Boolean).join(", "); + return detail ? `emptyDir (${detail})` : "emptyDir"; + } + const named = source as { name?: string; secretName?: string }; + const name = named?.name ?? named?.secretName; + return name ? `${kind} (${name})` : kind; +} + +function taskMounts(task: CRDJobTask | undefined): JobMount[] { + const container = task?.workload.template?.spec?.containers?.find( + (item) => item.name === "main", + ); + const volumes = task?.workload.template?.spec?.volumes ?? []; + return (container?.volumeMounts ?? []).map((vm) => + toJobMount( + volumes.find((v) => v.name === vm.name), + vm.mountPath, + ), + ); +} + export function crdToJob(crd: CRDJob): Job { const tasks = crd.spec?.tasks ?? []; const mainContainer = (task?: CRDJobTask) => @@ -45,8 +139,10 @@ export function crdToJob(crd: CRDJob): Job { task.name ?? ""; const roles = tasks.map(taskRoleName); - const roleCount = new Set(tasks.map((task) => task.role).filter(Boolean)) - .size; + // 角色数量必须与详情页和任务编辑器使用的角色名一致。`task.role` 只是用于分组 + // 展示的粗粒度分类（Actor/Rollout/Env），多个业务角色会被折叠成同一个取值， + // 据此计数会少报角色数。 + const roleCount = new Set(roles.filter(Boolean)).size; const displayName = crd.metadata?.annotations?.["rlark.io/display-name"] ?? crd.metadata?.labels?.["rlark.io/display-name"] ?? @@ -70,40 +166,11 @@ export function crdToJob(crd: CRDJob): Job { key: e.name, value: e.value ?? "", })) ?? []; - const taskMounts = (c?.volumeMounts ?? []).map((vm) => { - const vol = t.workload.template?.spec?.volumes?.find( - (v) => v.name === vm.name, - ); - if (vol?.ephemeral) { - const spec = vol.ephemeral.volumeClaimTemplate?.spec; - return { - type: "storage" as const, - objectStorage: spec?.storageClassName ?? "", - mountPath: vm.mountPath, - hostPath: "", - pvcSizeGb: storageGi(spec?.resources?.requests?.storage), - }; - } - if (vol?.persistentVolumeClaim) { - return { - type: "storage" as const, - objectStorage: vol.persistentVolumeClaim.claimName, - mountPath: vm.mountPath, - hostPath: "", - pvcSizeGb: 10, - }; - } - const hostPath = vol?.hostPath?.path ?? ""; - return { - type: "host" as const, - objectStorage: "", - mountPath: vm.mountPath, - hostPath, - pvcSizeGb: 10, - }; - }); + const taskMountList = taskMounts(t); return { role: taskRoleName(t), + // Job CRD 的 task 不携带集群标识，集群由 CreateJob 页面按 nodeSelector + // 反查节点所属 namespace 推断，这里只能留空。 cluster: "", nodeSelector: nsStr, replicas: carrierReplicas(t), @@ -114,7 +181,7 @@ export function crdToJob(crd: CRDJob): Job { image: c?.image ?? "", prepareScript: rayScript(t, "prepare"), env: taskEnv, - mounts: taskMounts, + mounts: taskMountList, }; }); const env = @@ -124,38 +191,7 @@ export function crdToJob(crd: CRDJob): Job { key: e.name, value: e.value ?? "", })) ?? []; - const mounts = (container?.volumeMounts ?? []).map((vm) => { - const vol = tasks[0]?.workload.template?.spec?.volumes?.find( - (v) => v.name === vm.name, - ); - if (vol?.ephemeral) { - const spec = vol.ephemeral.volumeClaimTemplate?.spec; - return { - type: "storage" as const, - objectStorage: spec?.storageClassName ?? "", - mountPath: vm.mountPath, - hostPath: "", - pvcSizeGb: storageGi(spec?.resources?.requests?.storage), - }; - } - if (vol?.persistentVolumeClaim) { - return { - type: "storage" as const, - objectStorage: vol.persistentVolumeClaim.claimName, - mountPath: vm.mountPath, - hostPath: "", - pvcSizeGb: 10, - }; - } - const hostPath = vol?.hostPath?.path ?? ""; - return { - type: "host" as const, - objectStorage: "", - mountPath: vm.mountPath, - hostPath, - pvcSizeGb: 10, - }; - }); + const mounts = taskMounts(tasks[0]); return { id: crd.metadata?.name ?? "", name: crd.metadata?.name ?? "", diff --git a/apps/rlark-ui/src/utils/job.ts b/apps/rlark-ui/src/utils/job.ts index 79ece44..876fe9a 100644 --- a/apps/rlark-ui/src/utils/job.ts +++ b/apps/rlark-ui/src/utils/job.ts @@ -196,6 +196,13 @@ export function generateJobCRD(opts: { const taskName = toResourceName(role); const hostMounts = roleMounts.filter((m) => m.type === "host"); const storageMounts = roleMounts.filter((m) => m.type === "storage"); + // 表单无法编辑的卷（emptyDir、configMap、secret 等）按原始定义写回， + // 只把 name 对齐到挂载路径派生的名字，保证 volumes 与 volumeMounts 匹配。 + const opaqueVolumes = roleMounts.flatMap((m) => + m.type === "opaque" && m.rawVolume + ? [{ ...m.rawVolume, name: toVolumeName(m.mountPath) }] + : [], + ); const containerVolumes = hostMounts.map((m) => ({ name: toVolumeName(m.mountPath), @@ -224,10 +231,13 @@ export function generateJobCRD(opts: { }; }); - const allVolumeMounts = roleMounts.map((m) => ({ - name: toVolumeName(m.mountPath), - mountPath: m.mountPath, - })); + // 丢失了原始定义的 opaque 挂载无法写出对应的 volume，跳过以免产生悬空引用。 + const allVolumeMounts = roleMounts + .filter((m) => m.type !== "opaque" || m.rawVolume) + .map((m) => ({ + name: toVolumeName(m.mountPath), + mountPath: m.mountPath, + })); return { name: taskName, @@ -300,6 +310,7 @@ export function generateJobCRD(opts: { volumes: [ ...(containerVolumes.length > 0 ? containerVolumes : []), ...(storageVolumes.length > 0 ? storageVolumes : []), + ...opaqueVolumes, ], }, }, diff --git a/apps/rlark-ui/src/utils/jobPhase.ts b/apps/rlark-ui/src/utils/jobPhase.ts index b373d49..35d1044 100644 --- a/apps/rlark-ui/src/utils/jobPhase.ts +++ b/apps/rlark-ui/src/utils/jobPhase.ts @@ -13,3 +13,16 @@ export function effectiveJobPhase(job: Job): JobDisplayPhase { } return job.phase || "Pending"; } + +// 终态集合：进入这些状态后任务不再运行，不应计入「运行中」。 +const TERMINAL_JOB_PHASES: readonly JobDisplayPhase[] = [ + "Stopped", + "Succeeded", + "Failed", + "Deleting", +]; + +// 判定任务是否仍处于活跃（非终态）状态，过渡态 Stopping 仍视为活跃。 +export function isActiveJob(job: Job): boolean { + return !TERMINAL_JOB_PHASES.includes(effectiveJobPhase(job)); +} diff --git a/apps/rlark-ui/src/utils/nodeResources.ts b/apps/rlark-ui/src/utils/nodeResources.ts index bb7fa5d..f301231 100644 --- a/apps/rlark-ui/src/utils/nodeResources.ts +++ b/apps/rlark-ui/src/utils/nodeResources.ts @@ -101,14 +101,19 @@ export function parseResourceQuantity( return value; } -export function formatResourceQuantity(key: string, raw?: string): string { +export function formatResourceQuantity( + key: string, + raw: string | undefined, + zh: boolean, +): string { const value = parseResourceQuantity(key, raw); if (value === null) return "—"; if (key === "memory" || key === "ephemeral-storage") { const gib = value / 1024 ** 3; return `${gib >= 100 ? gib.toFixed(0) : gib.toFixed(1)} GiB`; } - if (key === "cpu") return `${formatResourceNumber(value)} 核`; + if (key === "cpu") + return `${formatResourceNumber(value)} ${zh ? "核" : "cores"}`; return formatResourceNumber(value); } diff --git a/apps/rlark-ui/src/utils/nodes.ts b/apps/rlark-ui/src/utils/nodes.ts index d95460f..afc12c8 100644 --- a/apps/rlark-ui/src/utils/nodes.ts +++ b/apps/rlark-ui/src/utils/nodes.ts @@ -55,6 +55,27 @@ export function getNodeDeviceModel(node: CRDNode): string { ); } +// node.status.reason values emitted by the node controller. Unknown values are +// returned verbatim so server-provided reasons still surface in the UI. +const nodeStatusReasonLabels: Record = { + AgentLeaseActive: { zh: "节点代理心跳正常", en: "Agent lease active" }, + AgentLeaseExpired: { zh: "节点代理心跳已过期", en: "Agent lease expired" }, + NodeAgentLeaseMissing: { + zh: "缺少节点代理心跳", + en: "Node agent lease missing", + }, +}; + +export function nodeStatusReasonLabel( + reason: string | undefined, + zh: boolean, +): string { + if (!reason) return ""; + const label = nodeStatusReasonLabels[reason]; + if (!label) return reason; + return zh ? label.zh : label.en; +} + export const categoryLabels: Record< NodeCategory, { zh: string; en: string; icon: typeof CloudCog } diff --git a/apps/rlark-ui/src/utils/overlayContainer.ts b/apps/rlark-ui/src/utils/overlayContainer.ts new file mode 100644 index 0000000..fea4475 --- /dev/null +++ b/apps/rlark-ui/src/utils/overlayContainer.ts @@ -0,0 +1,15 @@ +/** + * Returns the DOM node that overlays (popovers, tooltips, dropdowns, modals) + * must be portalled into. + * + * Overlays are portalled so that their `position: fixed` box escapes the + * clipping and stacking contexts of the surrounding table or card. The portal + * target still has to live inside `.app-shell`, because that element carries + * the `theme-light` / `theme-dark` class which both supplies the theme CSS + * variables and scopes every `.theme-dark …` rule. Portalling into + * `document.body` puts the overlay outside that subtree, so it keeps the light + * palette while the rest of the page is dark. + */ +export function overlayContainer(): HTMLElement { + return document.querySelector(".app-shell") ?? document.body; +} diff --git a/apps/rlark-ui/src/utils/route.ts b/apps/rlark-ui/src/utils/route.ts index fab2d15..33b8875 100644 --- a/apps/rlark-ui/src/utils/route.ts +++ b/apps/rlark-ui/src/utils/route.ts @@ -43,13 +43,9 @@ export function parseAdminRoute( }; } -export function filesPath( - cluster: string, - storageClass: string, - admin = false, -) { +export function filesPath(storageClass: string, admin = false) { const prefix = admin ? "/admin/files" : "/files"; - return `${prefix}/${encodeURIComponent(cluster)}/${encodeURIComponent(storageClass)}`; + return `${prefix}/${encodeURIComponent(storageClass)}`; } export function hasTerminalSession(storage: Pick) { diff --git a/apps/rlark-ui/src/utils/sshKeys.ts b/apps/rlark-ui/src/utils/sshKeys.ts index f7a45e4..4cff163 100644 --- a/apps/rlark-ui/src/utils/sshKeys.ts +++ b/apps/rlark-ui/src/utils/sshKeys.ts @@ -74,3 +74,26 @@ export function resolveSSHKeyOwners( owners: ownersByKey.get(publicKey) ?? [], })); } + +/** + * Splits selected public keys into the ones still registered in SSH key + * management and the ones whose owner record was removed. A removed record + * cannot be rendered or deselected in the selection list, so a cloned or + * edited Job must not carry it forward silently. + */ +export function partitionSelectableSSHPublicKeys( + selected: string[], + knownKeys: SSHUserKey[], +): { known: string[]; orphaned: string[] } { + const registered = new Set( + groupSSHUserKeys(knownKeys).map(({ publicKey }) => publicKey), + ); + const known: string[] = []; + const orphaned: string[] = []; + + for (const publicKey of selected) { + (registered.has(publicKey.trim()) ? known : orphaned).push(publicKey); + } + + return { known, orphaned }; +} diff --git a/apps/rlark-ui/src/utils/workerEvents.ts b/apps/rlark-ui/src/utils/workerEvents.ts new file mode 100644 index 0000000..a37f9d3 --- /dev/null +++ b/apps/rlark-ui/src/utils/workerEvents.ts @@ -0,0 +1,121 @@ +import type { NodeEventEntry } from "../types"; + +// 一旦 Pod 已绑定节点，这些 reason 就只能是绑定之前产生的历史事件。 +// kube-scheduler 的 FailedScheduling 与卷绑定阶段的 FailedBinding 都只在 Pod +// 绑定前写入：使用 generic ephemeral volume 时首次调度必然先失败（PVC 尚未由 +// ephemeral volume controller 创建），约一秒后自动调度成功，因此必须按当前状态 +// 而不是事件流本身判断事件是否仍然有效。 +const RESOLVED_AFTER_SCHEDULING_REASONS = new Set([ + "FailedScheduling", + "FailedBinding", +]); + +// kubelet 的 Pulling/Pulled 是"某个容器镜像在某一刻的拉取进展"，不是 Pod 的 +// 持久状态。一个 Worker Pod 同时包含主容器与平台注入的 sidecar/init 容器， +// 每个容器各自上报一次，所以"镜像已拉取"很可能说的是 sidecar 镜像，而不是 +// 用户填写的业务镜像；把它与"正在拉取镜像""启动失败"并列展示会让用户以为 +// 错误镜像已经拉取成功。 +const PULL_PROGRESS_REASONS = new Set(["Pulling", "Pulled"]); + +// 镜像拉取处于失败状态时，任何历史拉取进展都不应再作为"当前等待原因"展示。 +const IMAGE_PULL_FAILURE_REASONS = new Set([ + "ErrImagePull", + "ImagePullBackOff", + "InvalidImageName", + "FailedToRetrieveImagePullSecret", +]); + +// kubelet 对镜像拉取失败和容器启动失败都使用 reason=Failed，仅凭 reason 会把 +// "镜像拉取失败"显示成"Worker 启动失败"。message 才带有可区分的信息，因此按 +// message 把 Failed 收敛到更精确的 reason；无法判定时保持原值。 +const INVALID_IMAGE_NAME_PATTERN = + /invalid reference format|couldn't parse image( reference)?|cannot parse image/i; +const IMAGE_PULL_FAILURE_PATTERN = + /failed to pull (and unpack )?image|errimagepull|imagepullbackoff|pull access denied|manifest unknown|manifest for .* not found|repository .* not found|does not exist or no pull access|unauthorized: authentication required/i; + +// workerEventTime 把事件时间解析为毫秒；缺失或非法时间返回 0，让排序与比较 +// 退化为"最旧"而不是抛出 NaN 污染比较结果。 +export function workerEventTime(event: NodeEventEntry): number { + const parsed = Date.parse(event.lastTime ?? ""); + return Number.isNaN(parsed) ? 0 : parsed; +} + +// refineWorkerEventReason 返回用于展示与去重的 reason。只收敛 kubelet 的 +// 通用 Failed 事件，其余 reason 原样返回。 +export function refineWorkerEventReason( + reason?: string, + message?: string, +): string { + const current = reason ?? ""; + if (current !== "Failed" || !message) return current; + if (INVALID_IMAGE_NAME_PATTERN.test(message)) return "InvalidImageName"; + if (IMAGE_PULL_FAILURE_PATTERN.test(message)) return "ErrImagePull"; + return current; +} + +// supersedePullProgress 从"当前有效"事件里剔除已经没有参考价值的镜像拉取 +// 进展：只保留最新的一条，并在出现不早于它的 Warning 事件、或镜像拉取已进入 +// 失败状态时整条剔除。被剔除的事件进入 resolved 分区，仍可排查。 +function supersedePullProgress( + active: NodeEventEntry[], + resolved: NodeEventEntry[], +): { active: NodeEventEntry[]; resolved: NodeEventEntry[] } { + const progress = active.filter((event) => + PULL_PROGRESS_REASONS.has(event.reason ?? ""), + ); + if (progress.length === 0) return { active, resolved }; + + const others = active.filter( + (event) => !PULL_PROGRESS_REASONS.has(event.reason ?? ""), + ); + const pullFailed = others.some((event) => + IMAGE_PULL_FAILURE_REASONS.has( + refineWorkerEventReason(event.reason, event.message), + ), + ); + const newestWarning = others.reduce( + (latest, event) => + event.type === "Normal" + ? latest + : Math.max(latest, workerEventTime(event)), + Number.NEGATIVE_INFINITY, + ); + const newestProgress = progress.reduce( + (latest, event) => + workerEventTime(event) > workerEventTime(latest) ? event : latest, + progress[0], + ); + const keep = + pullFailed || workerEventTime(newestProgress) < newestWarning + ? undefined + : newestProgress; + + const superseded = new Set(progress.filter((event) => event !== keep)); + return { + active: active.filter((event) => !superseded.has(event)), + resolved: [...resolved, ...superseded], + }; +} + +// partitionWorkerEvents 把 worker 事件拆成"当前有效"与"已被后续进展取代" +// 两组。Gateway 只放行 Warning 与 Pulling/Pulled 事件，Normal 的 Scheduled +// 事件拿不到，所以调用方以 scheduled（Pod 已分配节点）作为调度完成的判据。 +// resolved 事件不会被丢弃，调用方以弱化样式单独分区展示，保证可排查性。 +export function partitionWorkerEvents( + events: NodeEventEntry[], + scheduled: boolean, +): { active: NodeEventEntry[]; resolved: NodeEventEntry[] } { + const active: NodeEventEntry[] = []; + const resolved: NodeEventEntry[] = []; + for (const event of events) { + if ( + scheduled && + RESOLVED_AFTER_SCHEDULING_REASONS.has(event.reason ?? "") + ) { + resolved.push(event); + } else { + active.push(event); + } + } + return supersedePullProgress(active, resolved); +} diff --git a/apps/rlark-ui/tests/api.test.mjs b/apps/rlark-ui/tests/api.test.mjs index b28201a..7028575 100644 --- a/apps/rlark-ui/tests/api.test.mjs +++ b/apps/rlark-ui/tests/api.test.mjs @@ -14,6 +14,7 @@ import { } from "../dist/test/api.js"; import { apiReferenceApi, + clustersApi, domainsApi, nodesApi, systemConfigApi, @@ -101,6 +102,24 @@ test("request falls back to the raw body when it carries no detail", async () => assert.equal(errorMessage(error.body), ""); }); +test("cluster lifecycle actions use the gateway lifecycle route", async () => { + globalThis.sessionStorage = storage(); + globalThis.window = { dispatchEvent() {} }; + const calls = []; + globalThis.fetch = async (input, init) => { + calls.push({ input: String(input), init }); + return new Response(JSON.stringify({ data: {} }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }; + + await clustersApi.lifecycle("cluster/a", "delete"); + + assert.equal(calls[0].input, "/api/v1/clusters/cluster%2Fa/delete"); + assert.equal(calls[0].init.method, "POST"); +}); + test("resource APIs encode paths, query values, and JSON bodies", async () => { globalThis.sessionStorage = storage(); globalThis.window = { dispatchEvent() {} }; diff --git a/apps/rlark-ui/tests/clipboard.test.mjs b/apps/rlark-ui/tests/clipboard.test.mjs new file mode 100644 index 0000000..a5d42c8 --- /dev/null +++ b/apps/rlark-ui/tests/clipboard.test.mjs @@ -0,0 +1,224 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { copyText } from "../dist/test/utils/clipboard.js"; + +const sshKeysSource = await readFile( + new URL("../src/pages/SSHKeys.tsx", import.meta.url), + "utf8", +); + +// 控制台里所有复制入口都必须经过 utils/clipboard 的 copyText，否则在 HTTP +// 部署（非安全上下文）下 navigator.clipboard 为 undefined，复制会直接失效。 +const copyEntrySources = Object.fromEntries( + await Promise.all( + [ + "pages/Api.tsx", + "pages/Clusters.tsx", + "pages/Jobs.tsx", + "pages/SSHKeys.tsx", + "pages/SystemConfig.tsx", + "pages/Workflows.tsx", + "admin/CreateCluster.tsx", + ].map(async (relative) => [ + relative, + await readFile(new URL(`../src/${relative}`, import.meta.url), "utf8"), + ]), + ), +); + +// installDom 用最小 DOM/navigator 替身模拟两类部署： +// clipboard 为 null 表示非安全上下文（HTTP 访问控制台），此时浏览器不暴露 +// navigator.clipboard；execCommandResult 控制回退路径的返回值。 +function installDom({ clipboard = null, execCommandResult = true } = {}) { + const calls = { writes: [], commands: [], appended: 0, removed: 0 }; + const element = { + value: "", + style: {}, + focus() {}, + select() {}, + setSelectionRange() {}, + remove() { + calls.removed += 1; + }, + }; + const originalDocument = globalThis.document; + const originalNavigator = Object.getOwnPropertyDescriptor( + globalThis, + "navigator", + ); + + globalThis.document = { + createElement: () => element, + body: { + appendChild() { + calls.appended += 1; + }, + }, + execCommand(command) { + calls.commands.push({ command, value: element.value }); + return execCommandResult; + }, + }; + Object.defineProperty(globalThis, "navigator", { + value: clipboard + ? { + clipboard: { + async writeText(value) { + return clipboard(value, calls); + }, + }, + } + : {}, + configurable: true, + writable: true, + }); + + return { + calls, + restore() { + if (originalDocument === undefined) delete globalThis.document; + else globalThis.document = originalDocument; + if (originalNavigator) { + Object.defineProperty(globalThis, "navigator", originalNavigator); + } else { + delete globalThis.navigator; + } + }, + }; +} + +test("安全上下文下 copyText 走 Clipboard API", async () => { + const dom = installDom({ + clipboard: (value, calls) => { + calls.writes.push(value); + }, + }); + try { + assert.equal(await copyText("ssh-ed25519 AAAA"), true); + assert.deepEqual(dom.calls.writes, ["ssh-ed25519 AAAA"]); + assert.equal(dom.calls.commands.length, 0); + } finally { + dom.restore(); + } +}); + +test("navigator.clipboard 缺失时回退到 execCommand", async () => { + const dom = installDom(); + try { + assert.equal(await copyText("ssh-rsa BBBB"), true); + assert.deepEqual(dom.calls.commands, [ + { command: "copy", value: "ssh-rsa BBBB" }, + ]); + // 临时 textarea 必须挂载后再移除，避免污染页面。 + assert.equal(dom.calls.appended, 1); + assert.equal(dom.calls.removed, 1); + } finally { + dom.restore(); + } +}); + +test("Clipboard API 抛错时同样回退到 execCommand", async () => { + const dom = installDom({ + clipboard: () => { + throw new Error("NotAllowedError"); + }, + }); + try { + assert.equal(await copyText("ssh-ed25519 CCCC"), true); + assert.deepEqual(dom.calls.commands, [ + { command: "copy", value: "ssh-ed25519 CCCC" }, + ]); + } finally { + dom.restore(); + } +}); + +test("回退路径失败时 copyText 返回 false", async () => { + const dom = installDom({ execCommandResult: false }); + try { + assert.equal(await copyText("ssh-ed25519 DDDD"), false); + assert.equal(dom.calls.removed, 1); + } finally { + dom.restore(); + } +}); + +test("空文本不触碰剪贴板", async () => { + const dom = installDom(); + try { + assert.equal(await copyText(""), false); + assert.equal(dom.calls.commands.length, 0); + assert.equal(dom.calls.appended, 0); + } finally { + dom.restore(); + } +}); + +test("SSH 公钥页的两处复制都使用带回退的 copyText", () => { + assert.doesNotMatch(sshKeysSource, /navigator\.clipboard/); + assert.match(sshKeysSource, /import \{ copyText \} from "\.\.\/utils\/clipboard"/); + assert.match(sshKeysSource, /if \(!\(await copyText\(sshCommand\)\)\) return;/); + assert.match(sshKeysSource, /if \(!\(await copyText\(publicKey\)\)\) return;/); +}); + +test("SSH 公钥页复制成功后复用 copy-confirmed 动画", () => { + assert.match( + sshKeysSource, + /copied \? "icon-button copy-confirmed" : "icon-button"/, + ); + assert.match( + sshKeysSource, + /isCopied\s*\n\s*\? "icon-button copy-confirmed"\s*\n\s*: "icon-button"/, + ); +}); + +test("所有复制入口都改用 copyText 且不再直接调用 navigator.clipboard", () => { + for (const [relative, source] of Object.entries(copyEntrySources)) { + assert.doesNotMatch( + source, + /navigator\.clipboard/, + `${relative} 仍直接调用 navigator.clipboard`, + ); + assert.match( + source, + /import \{ copyText \} from "\.\.\/utils\/clipboard"/, + `${relative} 未引入 copyText`, + ); + } +}); + +test("复制成功态只在 copyText 返回 true 时置位", () => { + // Api / SystemConfig / Clusters / CreateCluster / Workflows / Jobs 都必须先 + // 判定复制结果，避免在回退也失败时仍显示"已复制"。 + assert.match(copyEntrySources["pages/Api.tsx"], /if \(!ok\) return;/); + assert.match( + copyEntrySources["pages/SystemConfig.tsx"], + /if \(!\(await copyText\(sshCommand\)\)\) return;/, + ); + assert.equal( + copyEntrySources["pages/Clusters.tsx"].match( + /if \(!\(await copyText\(sshCommand\)\)\) return;/g, + )?.length, + 2, + ); + assert.match( + copyEntrySources["admin/CreateCluster.tsx"], + /if \(!\(await copyText\(deployYaml\)\)\) return;/, + ); + assert.match( + copyEntrySources["admin/CreateCluster.tsx"], + /if \(!\(await copyText\(yaml\)\)\) return;/, + ); + assert.match( + copyEntrySources["pages/Jobs.tsx"], + /if \(!\(await copyText\(lines\.join\("\\n"\)\)\)\) return;/, + ); +}); + +test("工作流 YAML 复制补齐了成功反馈", () => { + const source = copyEntrySources["pages/Workflows.tsx"]; + assert.match(source, /if \(!\(await copyText\(yaml\)\)\) return;/); + assert.match(source, /setYamlCopied\(true\);/); + assert.match(source, /yamlCopied \? \(zh \? "已复制" : "Copied"\) : c\.api\.copy/); +}); diff --git a/apps/rlark-ui/tests/cluster-lifecycle-actions.test.mjs b/apps/rlark-ui/tests/cluster-lifecycle-actions.test.mjs new file mode 100644 index 0000000..76f0d59 --- /dev/null +++ b/apps/rlark-ui/tests/cluster-lifecycle-actions.test.mjs @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +const source = await readFile( + new URL("../src/pages/ClusterManagement.tsx", import.meta.url), + "utf8", +); + +test("delisted clusters expose restore and use the delist action", () => { + assert.match( + source, + /cluster\.phase === "Delisted" \? \([\s\S]{0,500}handleClusterAction\(cluster, "restore"\)/, + ); + assert.match(source, /handleClusterAction\(cluster, "delist"\)/); + assert.doesNotMatch(source, /handleClusterAction\(cluster\.id, "offline"\)/); +}); diff --git a/apps/rlark-ui/tests/form-modal-dismiss.test.mjs b/apps/rlark-ui/tests/form-modal-dismiss.test.mjs new file mode 100644 index 0000000..2668269 --- /dev/null +++ b/apps/rlark-ui/tests/form-modal-dismiss.test.mjs @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +const read = (relative) => readFile(new URL(relative, import.meta.url), "utf8"); + +// 数据录入类弹层：误点遮罩会丢失全部已填内容，因此遮罩不得关闭弹层。 +const formModals = [ + "../src/pages/CreateJob.tsx", + "../src/pages/Workflows.tsx", + "../src/pages/Storage.tsx", + "../src/pages/ImageRegistries.tsx", + "../src/pages/SSHKeys.tsx", + "../src/pages/Domains.tsx", +]; + +for (const relative of formModals) { + test(`${relative} keeps form modals open when the backdrop is clicked`, async () => { + const source = await read(relative); + const backdrops = source.matchAll( + /className="modal-backdrop[^"]*"([\s\S]{0,220}?)>/g, + ); + for (const [, tail] of backdrops) { + assert.doesNotMatch(tail, /onMouseDown/); + assert.doesNotMatch(tail, /onClick/); + } + }); +} + +test("job tags editing keeps its modal open when the backdrop is clicked", async () => { + const source = await read("../src/pages/Jobs.tsx"); + const [, tail] = source.match( + /className="modal-backdrop job-tags-modal-backdrop"([\s\S]{0,220}?)>/, + ); + assert.doesNotMatch(tail, /onMouseDown/); +}); + +test("confirmation dialogs still dismiss on backdrop click", async () => { + const jobs = await read("../src/pages/Jobs.tsx"); + for (const backdrop of [ + "job-lifecycle-backdrop", + "delete-job-backdrop", + "restart-choice-backdrop", + ]) { + const [, tail] = jobs.match( + new RegExp(`className="modal-backdrop ${backdrop}"([\\s\\S]{0,220}?)>`), + ); + assert.match(tail, /onMouseDown/); + } + const clusters = await read("../src/pages/ClusterManagement.tsx"); + const [, clusterTail] = clusters.match( + /className="modal-backdrop job-lifecycle-backdrop"([\s\S]{0,220}?)>/, + ); + assert.match(clusterTail, /onMouseDown/); +}); diff --git a/apps/rlark-ui/tests/image-reference.test.mjs b/apps/rlark-ui/tests/image-reference.test.mjs index ea4bc88..27209f3 100644 --- a/apps/rlark-ui/tests/image-reference.test.mjs +++ b/apps/rlark-ui/tests/image-reference.test.mjs @@ -8,7 +8,10 @@ test("accepts valid container image references", () => { imageReferenceHasWhitespace("docker.io/library/ubuntu:22.04"), false, ); - assert.equal(imageReferenceHasWhitespace("registry/team/image@sha256:abc"), false); + assert.equal( + imageReferenceHasWhitespace("registry/team/image@sha256:abc"), + false, + ); }); test("rejects spaces and other whitespace in image references", () => { diff --git a/apps/rlark-ui/tests/job-actions.test.mjs b/apps/rlark-ui/tests/job-actions.test.mjs index 08c7525..957d784 100644 --- a/apps/rlark-ui/tests/job-actions.test.mjs +++ b/apps/rlark-ui/tests/job-actions.test.mjs @@ -123,6 +123,21 @@ test("job lifecycle actions use the shared in-app confirmation dialog", () => { assert.match(jobsSource, /清理后启动任务？/); }); +test("job list refresh does not fan out one tasks request per job", () => { + const start = jobsSource.indexOf("const fetchJobs = async ("); + assert.ok(start > 0); + const end = jobsSource.indexOf("useAutoRefresh(fetchJobs", start); + assert.ok(end > start); + const fetchJobsBody = jobsSource.slice(start, end); + assert.doesNotMatch(fetchJobsBody, /tasksApi\.list/); + assert.doesNotMatch(fetchJobsBody, /nodesApi\.get/); + assert.doesNotMatch(fetchJobsBody, /rlinf\.io\/job=/); + // 节点级缓存由详情页在拉取本 job 的 Task/Node 时一并构建 + assert.match(jobsSource, /setNodePullProgressMap\(nodeProgressMap\)/); + assert.match(jobsSource, /setNodeEventsMap\(nodeWarningEventsMap\)/); + assert.match(jobsSource, /setNodeDeviceModelMap\(deviceModelMap\)/); +}); + test("worker node shows the shared info tooltip in the warning color", () => { assert.match(jobsSource, /isDiskUsageWarning\(n\)/); assert.match( @@ -132,7 +147,7 @@ test("worker node shows the shared info tooltip in the warning color", () => { assert.match(jobsSource, /setDetailNodeDiskWarningMap\(diskWarningMap\)/); assert.match( jobsSource, - /detailNodeDiskWarningMap\[worker\.node\].*nodeDiskWarningMap\[worker\.node\]/s, + /diskWarning=\{\s*detailNodeDiskWarningMap\[worker\.node\] \?\? false\s*\}/s, ); assert.match(jobsSource, /variant="danger"/); assert.match(jobsSource, /className=\{`status-info\$\{variant/); @@ -220,18 +235,15 @@ test("worker pending tooltip shows readable reasons without raw Kubernetes messa assert.match(jobsSource, /en: "Volume binding failed"/); assert.match(jobsSource, /en: "Container repeatedly failed to start"/); assert.match(jobsSource, /en: "Node PID pressure"/); - assert.match(jobsSource, /workerEventReasonLabel\(ev\.reason, zh, failed\)/); + assert.match(jobsSource, /workerEventLabel\(ev, zh, failed\)/); assert.doesNotMatch(jobsSource, /className="event-message">\{ev\.message\}/); assert.match(jobsStyles, /width: min\(360px, calc\(100vw - 24px\)\)/); }); test("worker pending tooltip groups duplicate reasons and shows the latest four", () => { assert.match(jobsSource, /\.sort\(\(left, right\) =>/); - assert.match(jobsSource, /new Map\(/); - assert.match( - jobsSource, - /workerEventReasonLabel\(event\.reason, zh, failed\)/, - ); + assert.match(jobsSource, /new Map\(\)/); + assert.match(jobsSource, /workerEventLabel\(event, zh, failed\)/); assert.match(jobsSource, /\.slice\(0, 4\)/); assert.match(jobsSource, /recentEvents\.map/); }); @@ -252,7 +264,7 @@ test("worker failed tooltip reuses readable Kubernetes event reasons", () => { ); assert.match(jobsSource, /failed=\{worker\.phase === "Failed"\}/); assert.match(jobsSource, /failed[\s\S]*?"失败原因"[\s\S]*?"Failure Reasons"/); - assert.match(jobsSource, /workerEventReasonLabel\(ev\.reason, zh, failed\)/); + assert.match(jobsSource, /workerEventLabel\(ev, zh, failed\)/); assert.match(jobsSource, /if \(failed\) return zh \? "Worker 运行失败"/); assert.match( jobsSource, diff --git a/apps/rlark-ui/tests/job-clone-fidelity.test.mjs b/apps/rlark-ui/tests/job-clone-fidelity.test.mjs new file mode 100644 index 0000000..5684954 --- /dev/null +++ b/apps/rlark-ui/tests/job-clone-fidelity.test.mjs @@ -0,0 +1,139 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { generateJobCRD } from "../dist/test/utils/job.js"; +import { crdToJob, describeVolumeSource } from "../dist/test/utils/crd.js"; + +const SHM_VOLUME = { + name: "shm", + emptyDir: { medium: "Memory", sizeLimit: "32Gi" }, +}; + +function actorResource(mounts) { + return { + role: "actor", + cluster: "cluster-a", + nodeSelector: "", + replicas: 1, + cpu: "8", + memory: "64Gi", + gpu: "4", + devices: [], + image: "busybox:latest", + prepareScript: "", + envs: [], + mounts, + }; +} + +function buildJobCRD(mounts) { + return generateJobCRD({ + name: "jo-0123456789abcdef", + type: "Custom", + headerRole: "actor", + roles: ["actor"], + roleResources: { actor: actorResource(mounts) }, + runScript: "echo ready", + domain: "", + }); +} + +// 模拟通过 Gateway 接口创建的任务：Pod 模板里带一个表单无法表达的 emptyDir 卷。 +function crdWithSharedMemory() { + const crd = buildJobCRD([ + { + type: "host", + objectStorage: "", + mountPath: "/mnt/data", + hostPath: "/data", + pvcSizeGb: 10, + }, + ]); + const podSpec = crd.spec.tasks[0].workload.template.spec; + podSpec.volumes.push(SHM_VOLUME); + podSpec.containers[0].volumeMounts.push({ + name: SHM_VOLUME.name, + mountPath: "/dev/shm", + }); + return crd; +} + +function cloneRoleResource(resource) { + return { + ...resource, + envs: resource.env, + prepareScript: resource.prepareScript ?? "", + }; +} + +test("maps volumes the form cannot edit to opaque mounts", () => { + const job = crdToJob(crdWithSharedMemory()); + + assert.deepEqual(job.resources[0].mounts[0], { + type: "host", + objectStorage: "", + mountPath: "/mnt/data", + hostPath: "/data", + pvcSizeGb: 10, + }); + const opaque = job.resources[0].mounts[1]; + assert.equal(opaque.type, "opaque"); + assert.equal(opaque.mountPath, "/dev/shm"); + assert.deepEqual(opaque.rawVolume, SHM_VOLUME); +}); + +test("keeps cpu and memory requests when reading a job CRD", () => { + const job = crdToJob(crdWithSharedMemory()); + + assert.equal(job.resources[0].cpu, "8"); + assert.equal(job.resources[0].memory, "64Gi"); +}); + +test("writes opaque volumes back verbatim when a job is cloned", () => { + const source = crdToJob(crdWithSharedMemory()); + const resource = source.resources[0]; + + const crd = generateJobCRD({ + name: "jo-fedcba9876543210", + type: "Custom", + headerRole: resource.role, + roles: [resource.role], + roleResources: { [resource.role]: cloneRoleResource(resource) }, + runScript: "echo ready", + domain: "", + }); + + const podSpec = crd.spec.tasks[0].workload.template.spec; + const shm = podSpec.volumes.find((volume) => volume.emptyDir); + assert.deepEqual(shm.emptyDir, SHM_VOLUME.emptyDir); + assert.ok( + podSpec.containers[0].volumeMounts.some( + (mount) => mount.name === shm.name && mount.mountPath === "/dev/shm", + ), + "the opaque volume must stay mounted at its original path", + ); +}); + +test("skips opaque mounts that lost their original volume definition", () => { + const crd = buildJobCRD([ + { + type: "opaque", + objectStorage: "", + mountPath: "/dev/shm", + hostPath: "", + pvcSizeGb: 10, + }, + ]); + + const podSpec = crd.spec.tasks[0].workload.template.spec; + assert.deepEqual(podSpec.volumes, []); + assert.equal(podSpec.containers[0].volumeMounts, undefined); +}); + +test("summarises opaque volume sources for read-only display", () => { + assert.equal(describeVolumeSource(SHM_VOLUME), "emptyDir (Memory, 32Gi)"); + assert.equal( + describeVolumeSource({ name: "cfg", configMap: { name: "train-config" } }), + "configMap (train-config)", + ); + assert.equal(describeVolumeSource(undefined), ""); +}); diff --git a/apps/rlark-ui/tests/job-header-role.test.mjs b/apps/rlark-ui/tests/job-header-role.test.mjs new file mode 100644 index 0000000..d21c91b --- /dev/null +++ b/apps/rlark-ui/tests/job-header-role.test.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { readStyles } from "./read-styles.mjs"; + +const createJobSource = await readFile( + new URL("../src/pages/CreateJob.tsx", import.meta.url), + "utf8", +); + +test("only single-pod roles can be picked as the ray header", () => { + assert.match( + createJobSource, + /const canBeHeader = \(role: string\) =>\s*roleReplicas\(role\) === 1;/, + ); + assert.match( + createJobSource, + /const selectHeaderRole = \(role: string\) => \{\s*if \(!canBeHeader\(role\)\) return;/, + ); + // Both the role step and the shared-config step must route through the guard + // instead of calling setHeaderRole directly. + assert.equal( + createJobSource.match(/onClick=\{\(\) => selectHeaderRole\(entry\.id\)\}/g) + ?.length, + 2, + ); + assert.equal( + createJobSource.match(/const headerAllowed = canBeHeader\(entry\.id\);/g) + ?.length, + 2, + ); + assert.equal( + createJobSource.match(/disabled=\{!headerAllowed\}/g)?.length, + 1, + ); +}); + +test("an invalidated header falls back to the first single-pod role", () => { + assert.match( + createJobSource, + /const effectiveHeader = canBeHeader\(selectedHeader\)\s*\? selectedHeader\s*: \(roles\.find\(\(entry\) => canBeHeader\(entry\.id\)\)\?\.id \?\? selectedHeader\);/, + ); + // The validation that bounces the wizard back to a header-less step reads the + // fallback value, so a recoverable state never deadlocks the wizard. + assert.match( + createJobSource, + /if \(effectiveHeader && !canBeHeader\(effectiveHeader\)\)/, + ); +}); + +test("multi-pod roles explain why they cannot become the header", () => { + assert.match( + createJobSource, + /Header 角色只能有一个 Pod，请将该角色副本数设为 1。/, + ); + assert.match( + createJobSource, + /The header role must have exactly one pod\. Set its replicas to 1\./, + ); + assert.match(createJobSource, /"Worker（多 Pod）"/); + assert.match(createJobSource, /"Worker \(multi-pod\)"/); +}); + +test("dark mode keeps the role name readable while typing", async () => { + const styles = await readStyles(); + assert.match( + styles, + /\.theme-dark \.role-edit-row input:focus \{[^}]*background: #0f1725;/, + ); + assert.match( + styles, + /\.theme-dark \.role-edit-row input::placeholder \{[^}]*color: #7c8798;/, + ); + assert.match( + styles, + /\.theme-dark \.role-edit-row small \{[^}]*color: #9ca8ba;/, + ); +}); diff --git a/apps/rlark-ui/tests/job-phase.test.mjs b/apps/rlark-ui/tests/job-phase.test.mjs index 1a2c18c..7121a65 100644 --- a/apps/rlark-ui/tests/job-phase.test.mjs +++ b/apps/rlark-ui/tests/job-phase.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { effectiveJobPhase } from "../dist/test/utils/jobPhase.js"; +import { effectiveJobPhase, isActiveJob } from "../dist/test/utils/jobPhase.js"; function job(phase, stopped, taskPhases) { return { @@ -43,7 +43,10 @@ test("distinguishes stopping from normal Pending states", () => { }); test("deleting takes precedence over stopping", () => { - assert.equal(effectiveJobPhase(job("Deleting", true, ["Stopped"])), "Deleting"); + assert.equal( + effectiveJobPhase(job("Deleting", true, ["Stopped"])), + "Deleting", + ); }); test("does not infer terminal states from task phases", () => { @@ -56,3 +59,18 @@ test("does not infer terminal states from task phases", () => { "Pending", ); }); + +test("isActiveJob keeps running and stopping jobs, drops terminal ones", () => { + assert.equal(isActiveJob(job("Running", false, ["Running"])), true); + assert.equal(isActiveJob(job("Pending", false, [])), true); + assert.equal(isActiveJob(job("Stopped", true, ["Stopped"])), false); + assert.equal(isActiveJob(job("Succeeded", false, ["Succeeded"])), false); + assert.equal(isActiveJob(job("Failed", false, ["Failed"])), false); + assert.equal(isActiveJob(job("Deleting", true, ["Stopped"])), false); +}); + +test("isActiveJob treats the Stopping transition as still active", () => { + const stopping = job("Running", true, ["Stopped", "Running"]); + assert.equal(effectiveJobPhase(stopping), "Stopping"); + assert.equal(isActiveJob(stopping), true); +}); diff --git a/apps/rlark-ui/tests/job-role-count-and-copy.test.mjs b/apps/rlark-ui/tests/job-role-count-and-copy.test.mjs new file mode 100644 index 0000000..64e0bd8 --- /dev/null +++ b/apps/rlark-ui/tests/job-role-count-and-copy.test.mjs @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { generateJobCRD } from "../dist/test/utils/job.js"; +import { crdToJob } from "../dist/test/utils/crd.js"; +import { readStyles } from "./read-styles.mjs"; + +const createJobSource = await readFile( + new URL("../src/pages/CreateJob.tsx", import.meta.url), + "utf8", +); +const jobsSource = await readFile( + new URL("../src/pages/Jobs.tsx", import.meta.url), + "utf8", +); + +function roleResource(role) { + return { + role, + cluster: "cluster-a", + nodeSelector: "", + replicas: 1, + cpu: "8", + memory: "64Gi", + gpu: "1", + devices: [], + image: "busybox:latest", + prepareScript: "", + envs: [], + mounts: [], + }; +} + +// 四个业务角色里 Actor/t1/t2 都会被 mapTaskRole 折叠成 "Actor"， +// Env 折叠成 "Env"，因此 task.role 只剩两个不同取值。 +const FOUR_ROLES = ["Actor", "Env", "t1", "t2"]; + +test("crdToJob 的角色数量按角色名统计，而不是按粗粒度分类", () => { + const crd = generateJobCRD({ + name: "jo-e89a97ca9a5c46ff", + type: "Custom", + headerRole: "Actor", + roles: FOUR_ROLES, + roleResources: Object.fromEntries( + FOUR_ROLES.map((role) => [role, roleResource(role)]), + ), + runScript: "echo ready", + domain: "", + }); + + const coarseRoles = new Set(crd.spec.tasks.map((task) => task.role)); + assert.ok( + coarseRoles.size < FOUR_ROLES.length, + "task.role 应当确实把多个业务角色折叠成更少的分类", + ); + + const job = crdToJob(crd); + assert.equal(job.roleCount, FOUR_ROLES.length); + assert.deepEqual(job.defaultRoles, FOUR_ROLES); +}); + +test("任务列表的角色数量列直接渲染 crdToJob 的 roleCount", () => { + assert.match(jobsSource, /\{job\.roleCount\}<\/td>/); +}); + +test("预填任务切换任务类型时不改写角色列表", () => { + const onTypeChange = createJobSource.slice( + createJobSource.indexOf("const onTypeChange = (next: JobType)"), + createJobSource.indexOf("const addRole = ("), + ); + assert.ok(onTypeChange.length > 0, "未找到 onTypeChange 实现"); + // 复制/编辑/编辑后重启场景（sourceJob 非空）必须在改写角色前提前返回。 + assert.match(onTypeChange, /if \(sourceJob\) return;/); + assert.ok( + onTypeChange.indexOf("if (sourceJob) return;") < + onTypeChange.indexOf("setRoles("), + "提前返回必须出现在 setRoles 之前", + ); + // 旧实现会把模板角色与已填镜像的预填角色拼接，导致角色重复累积。 + assert.doesNotMatch(onTypeChange, /keptEntries/); + assert.doesNotMatch(onTypeChange, /templateIds/); +}); + +test("角色运行配置的镜像复制按钮带复制成功反馈", () => { + assert.match(jobsSource, /const \[imageCopied, setImageCopied\] = useState/); + assert.match(jobsSource, /setImageCopied\(true\)/); + assert.match(jobsSource, /setImageCopied\(false\)/); + assert.match( + jobsSource, + /imageCopied \? "icon-button copy-confirmed" : "icon-button"/, + ); + assert.match(jobsSource, /imageCopied \? : { + const styles = await readStyles(); + assert.match(styles, /@keyframes icon-button-copy-pulse/); + assert.match( + styles, + /\.icon-button\.copy-confirmed\s*\{[^}]*animation: icon-button-copy-pulse/, + ); + assert.match(styles, /\.theme-dark \.icon-button\.copy-confirmed\s*\{/); + assert.match( + styles, + /@media \(prefers-reduced-motion: reduce\)\s*\{\s*\.icon-button\.copy-confirmed/, + ); +}); diff --git a/apps/rlark-ui/tests/job-ssh-keys.test.mjs b/apps/rlark-ui/tests/job-ssh-keys.test.mjs index a421486..f6d8fff 100644 --- a/apps/rlark-ui/tests/job-ssh-keys.test.mjs +++ b/apps/rlark-ui/tests/job-ssh-keys.test.mjs @@ -3,6 +3,7 @@ import test from "node:test"; import { findSSHKeyDuplicate, groupSSHUserKeys, + partitionSelectableSSHPublicKeys, resolveSSHKeyOwners, splitSSHPublicKeys, } from "../dist/test/utils/sshKeys.js"; @@ -71,6 +72,36 @@ test("keeps injected keys visible after their registered owner is removed", () = ]); }); +test("drops cloned public keys whose owner record was deleted", () => { + const knownKeys = [ + { index: 0, user: "bob", public_key: bobKey, added_at: "" }, + ]; + + assert.deepEqual( + partitionSelectableSSHPublicKeys([aliceKey, bobKey], knownKeys), + { known: [bobKey], orphaned: [aliceKey] }, + ); +}); + +test("keeps every cloned public key that is still registered", () => { + const knownKeys = [ + { index: 0, user: "alice", public_key: aliceKey, added_at: "" }, + { index: 0, user: "bob", public_key: bobKey, added_at: "" }, + ]; + + assert.deepEqual( + partitionSelectableSSHPublicKeys([aliceKey, bobKey], knownKeys), + { known: [aliceKey, bobKey], orphaned: [] }, + ); +}); + +test("drops every cloned public key when the registry is empty", () => { + assert.deepEqual(partitionSelectableSSHPublicKeys([aliceKey], []), { + known: [], + orphaned: [aliceKey], + }); +}); + test("preserves every selected SSH public key in the submitted Job", () => { const sshPublicKey = `${aliceKey}\n${bobKey}`; const roleResources = { diff --git a/apps/rlark-ui/tests/job-tag-filter.test.mjs b/apps/rlark-ui/tests/job-tag-filter.test.mjs new file mode 100644 index 0000000..9696179 --- /dev/null +++ b/apps/rlark-ui/tests/job-tag-filter.test.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { readStyles } from "./read-styles.mjs"; + +const popoverSource = await readFile( + new URL("../src/components/TagFilterPopover.tsx", import.meta.url), + "utf8", +); +const jobsSource = await readFile( + new URL("../src/pages/Jobs.tsx", import.meta.url), + "utf8", +); + +test("the tag filter still sends one key=value pair per selected value", () => { + // The gateway re-groups the flattened pairs by key, so the serialization has + // to stay a flat comma-separated list for the AND/OR split to apply. + assert.match( + jobsSource, + /Object\.entries\(tagFilter\)\s*\.flatMap\(\(\[key, values\]\) => values\.map\(\(value\) => `\$\{key\}=\$\{value\}`\)\)\s*\.join\(","\)/, + ); +}); + +test("the popover states that keys are ANDed and values are ORed", () => { + assert.match( + popoverSource, + /多个标签 key 之间为「且」，同一 key 下的多个值之间为「或」。/, + ); + assert.match( + popoverSource, + /Different keys are ANDed; values within the same key are ORed\./, + ); +}); + +test("no control selects every key at once", () => { + // Selecting all keys requires one job to carry all of them, which returns an + // empty list on any realistic data set. The only bulk control left operates + // on the values of a single key, where the backend ORs the selection. + assert.doesNotMatch(popoverSource, /toggleSelectAllKeys/); + assert.equal( + popoverSource.match(/\{zh \? "全选\/取消" : "All \/ None"\}/g)?.length, + 1, + ); + assert.match( + popoverSource, + /onClick=\{\(\) => toggleKeyAll\(effectiveKey\)\}/, + ); + // The bulk control is hidden until a key with values is in focus, otherwise + // it would have no target. + assert.match( + popoverSource, + /\{effectiveKey && selectedValues\.length > 0 && \(/, + ); +}); + +test("the key column keeps a static header instead of a bulk action", () => { + assert.match(popoverSource, /\{zh \? "标签 key" : "Keys"\}/); +}); + +test("only a click moves the focus between tag keys", () => { + // Hover-driven focus made the value column follow the pointer, so every key + // the pointer crossed on its way to the values replaced the column content + // and the values of the key below an already-selected one could never be + // ticked. The row is now a checkbox (select all) plus a button (focus). + assert.doesNotMatch(popoverSource, /onMouseEnter|onMouseLeave/); + assert.match( + popoverSource, + /className="tag-filter-key-main"\s*aria-pressed=\{effectiveKey === k\}\s*onClick=\{\(\) => setActiveKey\(k\)\}/, + ); + assert.match(popoverSource, /const \[activeKey, setActiveKey\] = useState/); + // The selected-key fallback only applies while no key is focused, or after a + // search removed the focused key from the list. + assert.match( + popoverSource, + /activeKey && filteredKeys\.includes\(activeKey\)\s*\?\s*activeKey\s*:\s*\(filteredKeys\.find\(keySelected\) \?\? null\)/, + ); + // The empty value column tells the user to click rather than to hover. + assert.match(popoverSource, /点击左侧的 key 查看对应的值/); + assert.match(popoverSource, /Click a key on the left to see its values/); +}); + +test("the focused key and the selected keys are styled apart", async () => { + const styles = await readStyles(); + // --blue-bg used to be undefined, so the selected rows fell back to a light + // lavender that stayed light in the dark theme and swallowed the label. + assert.match(styles, /\.theme-dark \{[^}]*--blue-bg:/); + assert.doesNotMatch(styles, /var\(--blue-bg,/); + assert.match( + styles, + /\.tag-filter-key-item\.active \{\s*border-right-color: var\(--blue\);/, + ); +}); + +test("the column header lays out the title and the bulk action side by side", async () => { + const styles = await readStyles(); + assert.match( + styles, + /\.tag-filter-col-head \{[^}]*justify-content: space-between;/, + ); + // Long tag keys must not push the bulk action out of the popover. + assert.match( + styles, + /\.tag-filter-col-title \{[^}]*text-overflow: ellipsis;/, + ); + assert.match(styles, /\.tag-filter-hint \{/); +}); diff --git a/apps/rlark-ui/tests/login-page.test.mjs b/apps/rlark-ui/tests/login-page.test.mjs index 1c15c7b..73282c0 100644 --- a/apps/rlark-ui/tests/login-page.test.mjs +++ b/apps/rlark-ui/tests/login-page.test.mjs @@ -12,16 +12,28 @@ const adminLoginSource = await readFile( ); test("user login shows the RLark brand logo", () => { - assert.match(loginSource, /src="\/rlark-logo-zh-light\.png"/); - assert.match(loginSource, /alt="RLark 具身智能云原生纳管平台"/); + assert.match( + loginSource, + /src=\{`\/rlark-logo-\$\{zh \? "zh" : "en"\}-light\.png`\}/, + ); + assert.match(loginSource, /"RLark 具身智能云原生纳管平台"/); + assert.match( + loginSource, + /"RLark cloud-native embodied intelligence platform"/, + ); }); test("password visibility control is accessible and does not submit", () => { assert.match(loginSource, /type=\{showPassword \? "text" : "password"\}/); - assert.match(loginSource, /placeholder="请输入密码"/); + assert.match( + loginSource, + /placeholder=\{zh \? "请输入密码" : "Enter your password"\}/, + ); assert.doesNotMatch(loginSource, /placeholder="•+/); assert.match(loginSource, /type="button"/); - assert.match(loginSource, /aria-label=\{showPassword \? "隐藏密码" : "显示密码"\}/); + assert.match(loginSource, /aria-label=\{\s*showPassword/); + assert.match(loginSource, /"隐藏密码"/); + assert.match(loginSource, /"Hide password"/); assert.match(loginSource, /aria-pressed=\{showPassword\}/); }); @@ -29,15 +41,27 @@ test("login errors render inline below the credentials", () => { assert.match(loginSource, /className="admin-login-panel"/); assert.match(loginSource, /className="login-inline-error"/); assert.match(loginSource, /role="alert"/); - assert.match(loginSource, /window\.setTimeout\(\(\) => setError\(""\), 4000\)/); + assert.match( + loginSource, + /window\.setTimeout\(\(\) => setError\(""\), 4000\)/, + ); }); test("admin login shares password and inline error controls", () => { assert.doesNotMatch(adminLoginSource, /className="admin-login-topbar"/); assert.match(adminLoginSource, /className="admin-login-panel"/); assert.match(adminLoginSource, /className="admin-login-badge">ADMIN/); - assert.match(adminLoginSource, /type=\{showPassword \? "text" : "password"\}/); - assert.match(adminLoginSource, /placeholder=\{zh \? "请输入密码" : "Enter password"\}/); + assert.match( + adminLoginSource, + /type=\{showPassword \? "text" : "password"\}/, + ); + assert.match( + adminLoginSource, + /placeholder=\{zh \? "请输入密码" : "Enter password"\}/, + ); assert.match(adminLoginSource, /className="login-inline-error"/); - assert.match(adminLoginSource, /window\.setTimeout\(\(\) => setError\(""\), 4000\)/); + assert.match( + adminLoginSource, + /window\.setTimeout\(\(\) => setError\(""\), 4000\)/, + ); }); diff --git a/apps/rlark-ui/tests/node-resources.test.mjs b/apps/rlark-ui/tests/node-resources.test.mjs index 986756c..007bb4d 100644 --- a/apps/rlark-ui/tests/node-resources.test.mjs +++ b/apps/rlark-ui/tests/node-resources.test.mjs @@ -11,11 +11,16 @@ import { test("formats Kubernetes binary quantities without inflating them", () => { assert.equal( - formatResourceQuantity("ephemeral-storage", "3748906852Ki"), + formatResourceQuantity("ephemeral-storage", "3748906852Ki", true), "3575 GiB", ); - assert.equal(formatResourceQuantity("memory", "1Gi"), "1.0 GiB"); - assert.equal(formatResourceQuantity("memory", "1000M"), "0.9 GiB"); + assert.equal(formatResourceQuantity("memory", "1Gi", true), "1.0 GiB"); + assert.equal(formatResourceQuantity("memory", "1000M", true), "0.9 GiB"); +}); + +test("localizes the CPU core unit", () => { + assert.equal(formatResourceQuantity("cpu", "8", true), "8 核"); + assert.equal(formatResourceQuantity("cpu", "8", false), "8 cores"); }); test("calculates disk usage against allocatable capacity", () => { diff --git a/apps/rlark-ui/tests/node-worker-localization.test.mjs b/apps/rlark-ui/tests/node-worker-localization.test.mjs new file mode 100644 index 0000000..dad8423 --- /dev/null +++ b/apps/rlark-ui/tests/node-worker-localization.test.mjs @@ -0,0 +1,97 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { readStyles } from "./read-styles.mjs"; + +const read = (path) => readFile(new URL(path, import.meta.url), "utf8"); + +test("maps node status reasons to localized labels", async () => { + const source = await read("../src/utils/nodes.ts"); + assert.match(source, /export function nodeStatusReasonLabel\(/); + assert.match(source, /AgentLeaseActive: \{ zh: "节点代理心跳正常"/); + assert.match(source, /AgentLeaseExpired: \{ zh: "节点代理心跳已过期"/); + assert.match(source, /NodeAgentLeaseMissing: \{/); + // Unknown reasons stay verbatim so new controller reasons remain visible. + assert.match(source, /if \(!label\) return reason;/); +}); + +test("renders node status reasons through the localized label", async () => { + const clusters = await read("../src/pages/Clusters.tsx"); + assert.match(clusters, /nodeStatusReasonLabel\(node\.status\.reason, zh\)/); + assert.doesNotMatch(clusters, /\{node\.status\.reason\}/); + + const overview = await read("../src/pages/Overview.tsx"); + assert.match(overview, /nodeStatusReasonLabel\(n\.status\?\.reason, isZh\)/); +}); + +test("keeps the node kind filter key stable across languages", async () => { + const jobs = await read("../src/pages/Jobs.tsx"); + assert.match(jobs, /function getNodeKind\(worker: WorkerItem\)/); + assert.match(jobs, /function nodeKindLabel\(kind: string, zh: boolean\)/); + assert.match(jobs, /set\.add\(getNodeKind\(w\)\)/); + assert.match(jobs, /workerKindFilter\.includes\(getNodeKind\(worker\)\)/); + assert.match(jobs, /nodeKindLabel\(getNodeKind\(worker\), zh\)/); +}); + +test("localizes addon phases reported by the controller", async () => { + const addons = await read("../src/admin/Addons.tsx"); + assert.match(addons, /const phaseLabel = \(phase: string\)/); + assert.match(addons, /if \(!zh\) return phase;/); + assert.match(addons, /phaseLabel\(a\.status\?\.phase \|\| "Pending"\)/); +}); + +test("passes the language into the terminal and login pages", async () => { + const terminal = await read("../src/components/terminal.tsx"); + assert.match(terminal, /const zh = lang === "zh";/); + assert.match(terminal, /function workerStatusLabel\(/); + + const login = await read("../src/pages/Login.tsx"); + assert.match(login, /const zh = lang === "zh";/); + assert.match(login, /rlark-logo-\$\{zh \? "zh" : "en"\}-light\.png/); + + const app = await read("../src/App.tsx"); + assert.match(app, / { + for (const path of [ + "../src/pages/Clusters.tsx", + "../src/pages/Storage.tsx", + "../src/admin/AdminPage.tsx", + "../src/components/NodeResourceBrowser.tsx", + ]) { + const source = await read(path); + // 中文顿号只允许出现在语言分支里：同一处必须给出英文分隔符的对应写法， + // 否则英文界面会直接显示中文标点。 + for (const match of source.matchAll(/join\("、"\)/g)) { + const context = source.slice(match.index, match.index + 160); + assert.match(context, /join\(", "\)/, `${path} @${match.index}`); + } + } +}); + +test("keeps the node worker table readable without horizontal scrolling", async () => { + const styles = await readStyles(); + assert.match( + styles, + /\.node-worker-table-wrap \.node-worker-table \{[^}]*min-width: 860px/, + ); + assert.match( + styles, + /\.node-worker-name \{[^}]*max-width: 200px[^}]*text-overflow: ellipsis/, + ); +}); + +test("truncates the long domain value in the pod sub-table", async () => { + const styles = await readStyles(); + assert.match(styles, /\.pod-subtable-truncated \{[^}]*max-width: 260px/); + assert.match( + styles, + /\.pod-subtable-truncated > span \{[^}]*text-overflow: ellipsis/, + ); + + const clusters = await read("../src/pages/Clusters.tsx"); + assert.match(clusters, /className="pod-subtable-truncated"/); + assert.match(clusters, /title=\{worker\.domain\}/); +}); diff --git a/apps/rlark-ui/tests/overlay-theme.test.mjs b/apps/rlark-ui/tests/overlay-theme.test.mjs new file mode 100644 index 0000000..2ac6395 --- /dev/null +++ b/apps/rlark-ui/tests/overlay-theme.test.mjs @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { readStyles } from "./read-styles.mjs"; + +const overlayContainerSource = await readFile( + new URL("../src/utils/overlayContainer.ts", import.meta.url), + "utf8", +); +const portalSources = await Promise.all( + ["../src/pages/Jobs.tsx", "../src/components/TagEditor.tsx"].map( + async (path) => [ + path, + await readFile(new URL(path, import.meta.url), "utf8"), + ], + ), +); + +test("overlays are portalled into the themed app shell, not the body", () => { + // `.app-shell` carries the theme-light / theme-dark class, which both + // supplies the theme CSS variables and scopes every `.theme-dark …` rule. + // Portalling into document.body puts the overlay outside that subtree, so it + // renders with the light palette while the rest of the page is dark. + assert.match( + overlayContainerSource, + /document\.querySelector\("\.app-shell"\) \?\? document\.body/, + ); +}); + +test("no component portals an overlay straight into document.body", () => { + for (const [path, source] of portalSources) { + for (const [, target] of source.matchAll( + /createPortal\(\s*(?:[\s\S]*?),\s*([^,()\s][^,]*?),?\s*\)/g, + )) { + assert.doesNotMatch( + target, + /document\.body/, + `${path} still portals an overlay into document.body`, + ); + } + } + for (const [path, source] of portalSources) { + assert.match( + source, + /import \{ overlayContainer \} from "\.\.\/utils\/overlayContainer";/, + `${path} must resolve its portal target through overlayContainer()`, + ); + } +}); + +test("overlay surfaces take their colours from the theme variables", async () => { + const styles = await readStyles(); + // The job tag popover used to hardcode a white background and a light grey + // border, which stayed light even once the portal target was fixed. + assert.match( + styles, + /\.job-tag-popover \{[^}]*background: var\(--panel\);[^}]*border: 1px solid var\(--line-strong\);/, + ); + assert.match( + styles, + /\.job-tag-chip \{[^}]*border: 1px solid var\(--line-strong\);/, + ); + assert.doesNotMatch(styles, /var\(--card-bg/); +}); + +test("light-only accent surfaces inside overlays have a dark-theme override", async () => { + const styles = await readStyles(); + assert.match(styles, /\.theme-dark \.tag-editor-error \{/); + assert.match(styles, /\.theme-dark \.job-tag-chip-inline \{/); +}); diff --git a/apps/rlark-ui/tests/page-layout.test.mjs b/apps/rlark-ui/tests/page-layout.test.mjs index 4f71af0..704105d 100644 --- a/apps/rlark-ui/tests/page-layout.test.mjs +++ b/apps/rlark-ui/tests/page-layout.test.mjs @@ -20,6 +20,17 @@ test("header-adjacent pages use the shared page shell", () => { ); }); +test("the overview live job list navigates to the job detail", () => { + assert.match(overview, /navigate\("jobs", job\.name\)/); +}); + +test("overview uses the same business worker filter as cluster management", () => { + assert.match( + overview, + /const displayNodes = realNodes\.filter\(isBusinessWorkerNode\);/, + ); +}); + test("page-specific classes do not override shared outer spacing", () => { assert.match( styles, @@ -72,3 +83,65 @@ test("page-specific classes do not override shared outer spacing", () => { ); } }); + +test("the job tag column stays on the row's middle line", () => { + // `td` inherits `vertical-align: middle` from the row, which is what keeps + // the type chip and the status badge centred. The tag cell used to opt out + // with `vertical-align: top` and floated above its neighbours. + const rule = styles.match(/\.job-table-tag-cell \{([\s\S]*?)\}/)?.[1]; + assert.ok(rule); + assert.doesNotMatch(rule, /vertical-align/); +}); + +const zIndexOf = (selector) => { + const rules = [ + ...styles.matchAll(new RegExp(`\\${selector} \\{([\\s\\S]*?)\\}`, "g")), + ]; + assert.ok(rules.length > 0, `${selector} rule not found`); + const values = rules + .map((rule) => rule[1].match(/z-index:\s*(-?\d+)/)?.[1]) + .filter((value) => value !== undefined) + .map(Number); + assert.ok(values.length > 0, `${selector} declares no z-index`); + return Math.max(...values); +}; + +test("full-screen overlays stack above the sticky topbar", () => { + const topbar = zIndexOf(".topbar"); + + for (const selector of [ + ".modal-backdrop", + ".addon-drawer-overlay", + ".addon-drawer", + ".log-terminal-fullscreen", + ]) { + assert.ok( + zIndexOf(selector) > topbar, + `${selector} must stack above .topbar (${topbar}) or it cannot cover the header`, + ); + } + + assert.ok(zIndexOf(".modal-backdrop") > zIndexOf(".action-dropdown")); + assert.ok(zIndexOf(".modal-backdrop") > zIndexOf(".log-terminal-fullscreen")); +}); + +test("modal backdrops do not need per-dialog z-index patches", () => { + for (const selector of [ + ".restart-choice-backdrop", + ".delete-job-backdrop", + ".job-lifecycle-backdrop", + ".storage-create-backdrop", + ".job-tags-modal-backdrop", + ]) { + const rules = [ + ...styles.matchAll(new RegExp(`\\${selector} \\{([\\s\\S]*?)\\}`, "g")), + ]; + for (const rule of rules) { + assert.doesNotMatch( + rule[1], + /z-index/, + `${selector} must inherit the shared .modal-backdrop stacking level`, + ); + } + } +}); diff --git a/apps/rlark-ui/tests/resource-placement.test.mjs b/apps/rlark-ui/tests/resource-placement.test.mjs index 13e5f73..1d98e47 100644 --- a/apps/rlark-ui/tests/resource-placement.test.mjs +++ b/apps/rlark-ui/tests/resource-placement.test.mjs @@ -16,25 +16,25 @@ test("lists nodes without discovered device resources", () => { }); test("shows a truthful placeholder when a device resource has no model", () => { - assert.match(pickerSource, /const deviceModel = getNodeDeviceModel\(fullNode\)/); - assert.match(pickerSource, /resourceKey: deviceModel \? deviceKey : ""/); - assert.match(pickerSource, /configured: Boolean\(deviceModel\)/); - assert.match(pickerSource, /"未设置设备型号"/); assert.match( pickerSource, - /仅按未设置设备型号的节点调度，不申请设备资源/, + /const deviceModel = getNodeDeviceModel\(fullNode\)/, ); + assert.match(pickerSource, /resourceKey: deviceModel \? deviceKey : ""/); + assert.match(pickerSource, /configured: Boolean\(deviceModel\)/); + assert.match(pickerSource, /"未设置设备型号"/); + assert.match(pickerSource, /仅按未设置设备型号的节点调度，不申请设备资源/); assert.match(pickerSource, /resourceAmount} × \$\{displayModel}/); }); test("unconfigured devices constrain nodes without requesting fake resources", () => { - assert.match(pickerSource, /!resource\.configured \|\| resource\.free >= nextPerWorker/); - assert.match(pickerSource, /candidates\[0\]\?\.resourceKey/); - assert.match(pickerSource, /disabled=\{isUnconfiguredDevice\}/); assert.match( pickerSource, - /仅按未分类节点调度，不申请未发现的设备资源/, + /!resource\.configured \|\| resource\.free >= nextPerWorker/, ); + assert.match(pickerSource, /candidates\[0\]\?\.resourceKey/); + assert.match(pickerSource, /disabled=\{isUnconfiguredDevice\}/); + assert.match(pickerSource, /仅按未分类节点调度，不申请未发现的设备资源/); }); test("edit mode restores the original resource and selected nodes", () => { @@ -49,11 +49,22 @@ test("edit mode restores the original resource and selected nodes", () => { assert.match(pickerSource, /initializedRef\.current = true/); }); +test("translates the unlabeled GPU placeholder instead of hardcoding Chinese", () => { + assert.match(pickerSource, /const UNLABELED_GPU = "__unlabeled_gpu__";/); + assert.match(pickerSource, /getNodeGPUModel\(fullNode\) \|\| UNLABELED_GPU/); + assert.doesNotMatch(pickerSource, /\|\| "未标注 GPU"/); + assert.match(pickerSource, /zh \? "未标注 GPU" : "Unlabeled GPU"/); + // Every display site must map the sentinel back to a localized label. + assert.equal(pickerSource.match(/=== UNLABELED_GPU/g)?.length, 3); +}); + test("new placement defaults to manual node selection", () => { - assert.match( - pickerSource, - /useState\(placementMode \?\? "manual"\)/, - ); + assert.match(pickerSource, /useState\(placementMode \?\? "manual"\)/); +}); + +test("per-worker summary punctuates by locale instead of reusing a full-width colon", () => { + assert.match(pickerSource, /zh \? "单 Worker：" : "Per worker: "/); + assert.doesNotMatch(pickerSource, /"Per worker"\}：/); }); test("node cards keep long node names readable", () => { @@ -61,3 +72,44 @@ test("node cards keep long node names readable", () => { assert.match(pickerSource, /className="placement-node-meta"/); assert.match(pickerSource, /className="placement-node-state"/); }); + +test("number fields stay clearable while editing", () => { + // 数字输入框必须经过 IntegerField，由 draft 保存编辑期的原始输入， + // 失焦后才归一化，否则删除最后一位会被立刻写回 min（0/1）而无法清空。 + assert.match(pickerSource, /function IntegerField\(/); + assert.match( + pickerSource, + /const \[draft, setDraft\] = useState\(null\)/, + ); + assert.match(pickerSource, /value=\{draft \?\? String\(value\)\}/); + assert.match(pickerSource, /onBlur=\{\(\) => setDraft\(null\)\}/); + assert.match( + pickerSource, + /raw\.trim\(\) === "" \|\| !Number\.isFinite\(parsed\)\s*\?\s*min/, + ); + // 两个数字输入框（单 Worker 资源数量、Worker 数量）都必须复用 IntegerField， + // 全文件只应保留 IntegerField 内部这一处裸 。 + assert.equal(pickerSource.match(/type="number"/g)?.length, 1); + assert.match(pickerSource, / { + // 改单 Worker 资源数量不得清空已选节点：只剔除空闲量不再满足新规格的节点。 + // 编辑期的空值会被 IntegerField 归一化成 min，中间态同样不能误伤选择。 + assert.match(pickerSource, /const selectionWithin = \(amount: number\) =>/); + assert.match( + pickerSource, + /selected\.has\(resource\.node\.metadata\.name\) &&\s*\(!resource\.configured \|\| resource\.free >= amount\)/, + ); + assert.match( + pickerSource, + /commitSelection\(selectionWithin\(nextAmount\), nextAmount\)/, + ); + // commitSelection 必须能带上新规格，否则 onChange 会回写旧的 gpu/device 数量。 + assert.match( + pickerSource, + /const commitSelection = \(next: Set, amount = resourceAmount\)/, + ); + assert.match(pickerSource, /gpu: kind === "gpu" \? String\(amount\) : "0"/); +}); diff --git a/apps/rlark-ui/tests/route.test.mjs b/apps/rlark-ui/tests/route.test.mjs index d1c0c27..844c770 100644 --- a/apps/rlark-ui/tests/route.test.mjs +++ b/apps/rlark-ui/tests/route.test.mjs @@ -8,15 +8,12 @@ import { parseAdminRoute, } from "../dist/test/utils/route.js"; -test("admin routes preserve shared file browser context", () => { - assert.equal(filesPath("cluster a", "shared/data"), "/files/cluster%20a/shared%2Fdata"); - assert.equal( - filesPath("cluster a", "shared/data", true), - "/admin/files/cluster%20a/shared%2Fdata", - ); - assert.deepEqual(parseAdminRoute("/admin/files/cluster%20a/shared%2Fdata"), { +test("admin routes preserve the storage class file browser context", () => { + assert.equal(filesPath("shared/data"), "/files/shared%2Fdata"); + assert.equal(filesPath("shared/data", true), "/admin/files/shared%2Fdata"); + assert.deepEqual(parseAdminRoute("/admin/files/shared%2Fdata"), { page: "files", - sub: "cluster a/shared/data", + sub: "shared/data", }); }); @@ -28,7 +25,10 @@ test("admin path matching observes a route boundary", () => { test("terminal requires an access token", () => { const storage = (values) => ({ getItem: (key) => values[key] ?? null }); - assert.equal(hasTerminalSession(storage({ "rlark-auth-token": "token" })), true); + assert.equal( + hasTerminalSession(storage({ "rlark-auth-token": "token" })), + true, + ); assert.equal(hasTerminalSession(storage({ "rlark-user-auth": "1" })), false); assert.equal(hasTerminalSession(storage({})), false); }); diff --git a/apps/rlark-ui/tests/worker-events.test.mjs b/apps/rlark-ui/tests/worker-events.test.mjs new file mode 100644 index 0000000..db1cd2b --- /dev/null +++ b/apps/rlark-ui/tests/worker-events.test.mjs @@ -0,0 +1,203 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { + partitionWorkerEvents, + refineWorkerEventReason, + workerEventTime, +} from "../dist/test/utils/workerEvents.js"; +import { readStyles } from "./read-styles.mjs"; + +const jobsSource = await readFile( + new URL("../src/pages/Jobs.tsx", import.meta.url), + "utf8", +); + +const failedScheduling = { + type: "Warning", + reason: "FailedScheduling", + message: + "0/3 nodes are available: waiting for ephemeral volume controller to create the persistentvolumeclaim", + lastTime: "2026-08-08T09:05:00Z", +}; +const diskPressure = { + type: "Warning", + reason: "NodeHasDiskPressure", + lastTime: "2026-08-08T09:06:00Z", +}; + +test("keeps every event active while the pod is still unscheduled", () => { + const { active, resolved } = partitionWorkerEvents( + [failedScheduling, diskPressure], + false, + ); + assert.deepEqual(active, [failedScheduling, diskPressure]); + assert.deepEqual(resolved, []); +}); + +test("moves pre-binding FailedScheduling to resolved once scheduled", () => { + const { active, resolved } = partitionWorkerEvents( + [failedScheduling, diskPressure], + true, + ); + assert.deepEqual(active, [diskPressure]); + assert.deepEqual(resolved, [failedScheduling]); +}); + +test("does not resolve events unrelated to scheduling", () => { + const { active, resolved } = partitionWorkerEvents([diskPressure], true); + assert.deepEqual(active, [diskPressure]); + assert.deepEqual(resolved, []); +}); + +test("tolerates entries without a reason", () => { + const nameless = { type: "Warning" }; + const { active, resolved } = partitionWorkerEvents([nameless], true); + assert.deepEqual(active, [nameless]); + assert.deepEqual(resolved, []); +}); + +test("worker status summary only reports FailedScheduling before binding", () => { + assert.match(jobsSource, /if \(schedulingFailed && !scheduled\)/); + assert.match( + jobsSource, + /if \(message === "FailedScheduling"\) return undefined;/, + ); + assert.match(jobsSource, /const scheduled = Boolean\(pod\.node\);/); + assert.match(jobsSource, /partitionWorkerEvents\(workerEvents, scheduled\)/); + assert.match(jobsSource, /events: activeWorkerEvents/); + assert.match(jobsSource, /resolvedEvents: resolvedWorkerEvents/); +}); + +test("status tooltip renders superseded events in a separate muted section", () => { + assert.match(jobsSource, /resolvedEvents = \[\]/); + assert.match(jobsSource, /const recentResolvedEvents = summarizeEvents\(/); + // 被取代的"镜像已拉取"并没有"恢复"，文案改成中性的历史事件。 + assert.match(jobsSource, /"历史事件" : "Earlier Events"/); + assert.doesNotMatch(jobsSource, /已恢复/); + assert.match(jobsSource, /event-chip event-resolved/); +}); + +const pulling = { + type: "Normal", + reason: "Pulling", + message: 'Pulling image "123"', + lastTime: "2026-08-08T09:47:15Z", +}; +const pulled = { + type: "Normal", + reason: "Pulled", + message: 'Successfully pulled image "rlark/network-sidecar:v1"', + lastTime: "2026-08-08T09:47:14Z", +}; +const pullFailed = { + type: "Warning", + reason: "Failed", + message: 'Failed to pull image "123": invalid reference format', + lastTime: "2026-08-08T09:47:16Z", +}; + +test("workerEventTime falls back to 0 for missing or invalid timestamps", () => { + assert.equal(workerEventTime(pulling), Date.parse(pulling.lastTime)); + assert.equal(workerEventTime({ type: "Normal" }), 0); + assert.equal(workerEventTime({ type: "Normal", lastTime: "nope" }), 0); +}); + +test("refineWorkerEventReason narrows the generic kubelet Failed reason", () => { + assert.equal( + refineWorkerEventReason("Failed", 'Failed to pull image "123": invalid reference format'), + "InvalidImageName", + ); + assert.equal( + refineWorkerEventReason( + "Failed", + 'Failed to pull image "a/b:c": manifest unknown', + ), + "ErrImagePull", + ); + // 真正的容器启动失败保持原样，仍然显示为 Worker 启动失败。 + assert.equal( + refineWorkerEventReason("Failed", 'failed to start container "worker"'), + "Failed", + ); + assert.equal(refineWorkerEventReason("Failed", undefined), "Failed"); + assert.equal(refineWorkerEventReason(undefined, "anything"), ""); + assert.equal(refineWorkerEventReason("BackOff", "pull image"), "BackOff"); +}); + +test("keeps only the newest image pull progress event", () => { + const { active, resolved } = partitionWorkerEvents([pulled, pulling], true); + assert.deepEqual(active, [pulling]); + assert.deepEqual(resolved, [pulled]); +}); + +test("supersedes every pull progress event once the image pull failed", () => { + const { active, resolved } = partitionWorkerEvents( + [pulled, pulling, pullFailed], + true, + ); + assert.deepEqual(active, [pullFailed]); + assert.deepEqual(resolved, [pulled, pulling]); +}); + +test("supersedes pull progress older than the newest warning", () => { + const staleWarning = { + type: "Warning", + reason: "BackOff", + message: "Back-off restarting failed container", + lastTime: "2026-08-08T09:48:00Z", + }; + const { active, resolved } = partitionWorkerEvents( + [pulling, staleWarning], + true, + ); + assert.deepEqual(active, [staleWarning]); + assert.deepEqual(resolved, [pulling]); +}); + +test("keeps pull progress that is newer than the last warning", () => { + const earlierWarning = { + type: "Warning", + reason: "FailedCreatePodSandBox", + lastTime: "2026-08-08T09:47:00Z", + }; + const { active, resolved } = partitionWorkerEvents( + [earlierWarning, pulling], + true, + ); + assert.deepEqual(active, [earlierWarning, pulling]); + assert.deepEqual(resolved, []); +}); + +test("worker event chips are deduped by label keeping the newest timestamp", () => { + assert.match(jobsSource, /const newest = new Map\(\)/); + assert.match( + jobsSource, + /workerEventTime\(event\) >= workerEventTime\(existing\)/, + ); + assert.match( + jobsSource, + /\.sort\(\(left, right\) => workerEventTime\(right\) - workerEventTime\(left\)\)/, + ); + assert.match(jobsSource, /\{workerEventLabel\(ev, zh, failed\)\}/); +}); + +test("worker event labels go through refineWorkerEventReason", () => { + assert.match(jobsSource, /refineWorkerEventReason\(event\.reason, event\.message\)/); +}); + +test("status tooltip clears the sticky topbar", async () => { + const styles = await readStyles(); + const tooltip = styles.match(/\.status-info-tooltip \{[^}]*\}/)?.[0] ?? ""; + const topbar = styles.match(/\.topbar \{[^}]*\}/)?.[0] ?? ""; + const tooltipZ = Number(tooltip.match(/z-index:\s*(\d+)/)?.[1]); + const topbarZ = Number(topbar.match(/z-index:\s*(\d+)/)?.[1]); + assert.ok(Number.isFinite(tooltipZ) && Number.isFinite(topbarZ)); + assert.ok( + tooltipZ > topbarZ, + `tooltip z-index ${tooltipZ} must stack above topbar ${topbarZ}`, + ); + assert.match(jobsSource, /document\.querySelector\("\.topbar"\)/); + assert.match(jobsSource, /const spaceAbove = iconRect\.top - topInset;/); + assert.match(jobsSource, /Math\.max\(\s*topInset \+ margin,/); +}); diff --git a/apps/rlark-ui/tsconfig.test.json b/apps/rlark-ui/tsconfig.test.json index ef7fdb0..f362ef4 100644 --- a/apps/rlark-ui/tsconfig.test.json +++ b/apps/rlark-ui/tsconfig.test.json @@ -13,6 +13,7 @@ "include": [ "src/api.ts", "src/backend.ts", + "src/utils/clipboard.ts", "src/utils/crd.ts", "src/utils/deployYaml.ts", "src/utils/imageReference.ts", @@ -24,6 +25,7 @@ "src/utils/resourceAvailability.ts", "src/utils/route.ts", "src/utils/sshKeys.ts", - "src/utils/terminalKeyboard.ts" + "src/utils/terminalKeyboard.ts", + "src/utils/workerEvents.ts" ] } diff --git a/apps/rlark/docs/admin-guide/agent.md b/apps/rlark/docs/admin-guide/agent.md index 48a2c7c..cec707e 100644 --- a/apps/rlark/docs/admin-guide/agent.md +++ b/apps/rlark/docs/admin-guide/agent.md @@ -130,17 +130,19 @@ A legacy cluster that has not reconnected is not guessed or synthesized. Keep it See [Configuration Reference](../reference/configuration.md#rlark-agent) for the complete list. -The Runtime/Workload coordinator is the production writer for Deployment, DaemonSet, and StatefulSet Tasks. It executes ordered Runtime operations. Runtime push controllers only enqueue reconciliation hints; verified targeted observation and Task Status updates run through the pull controller. +The Runtime/Workload coordinator is the production writer for Deployment and StatefulSet Tasks. It executes ordered Runtime operations. Runtime push controllers only enqueue reconciliation hints; verified targeted observation and Task Status updates run through the pull controller. + +The Ray-on-Kubernetes Workload Adapter supports Deployment and StatefulSet carriers. The cluster Agent runs one repair scan immediately after startup and repeats it at the configured jittered interval. Each scan pages through management Tasks assigned or eligible for the same Host with a 10,000-Task budget and emits full-reconciliation hints; it does not mutate Tasks or Runtime objects directly. Transient management API list failures are logged and retried in a later round without stopping the Agent. Trigger-bridge pressure is summarized but does not stop the scan from visiting later Tasks and pages. Disabling the scan removes this recovery path for lost watch events and should be done only when another operational mechanism guarantees convergence. -During an Agent upgrade, the coordinator recognizes primary workloads created by the previous Task controller even when they do not yet carry Task UID annotations. It adopts only a same-name Deployment, DaemonSet, or StatefulSet whose existing metadata does not conflict with the Task; this compatibility rule does not apply to arbitrary auxiliary resources. Initial adoption and subsequent reconciliation while the workload remains at that legacy adopted generation update identity and revision metadata only, preserving the running Pod template and avoiding an upgrade-triggered rollout. After a later Task generation establishes the workload as canonical-owned, full reconciliation repairs out-of-band drift in Agent-managed replicas and Pod-template fields even when the current Task generation has not changed. +During an Agent upgrade, the coordinator recognizes primary workloads created by the previous Task controller even when they do not yet carry Task UID annotations. It adopts only a same-name Deployment or StatefulSet whose existing metadata does not conflict with the Task; this compatibility rule does not apply to arbitrary auxiliary resources. Initial adoption and subsequent reconciliation while the workload remains at that legacy adopted generation update identity and revision metadata only, preserving the running Pod template and avoiding an upgrade-triggered rollout. After a later Task generation establishes the workload as canonical-owned, full reconciliation repairs out-of-band drift in Agent-managed replicas and Pod-template fields even when the current Task generation has not changed. Do not change `spec.runtime.kind` or `spec.runtime.name` to migrate an already claimed Task. The Agent freezes the claimed Runtime: such a change leaves the old workload in place, does not apply any other Spec changes from that generation, and waits for the original value to be restored or for the Task to be deleted and recreated. A decode, validation, or implementation-registration failure on a claimed, non-deleting Task also preserves the claim and existing workload instead of cleaning it up. Deletion still cleans up through the claimed Runtime and Task identity. During upgrade, a legacy Task that has the cleanup finalizer but no claimant annotation is adopted only when it is deleting or the current Agent can realize it; a non-deleting Task selected for another Agent is left untouched. See [Runtime, Workload, and Environment Boundary](../developer-guide/runtime-provider.md) for the ownership contract. -Upper-layer services should not change `spec.workload.kind` on an already claimed Kubernetes Task as their normal workflow. Replace the Task with a new UID—or use the Job restart/replacement workflow—to change carrier kind. If a direct edit nevertheless reaches the Agent, it invokes a narrow, recoverable compatibility path among Deployment, DaemonSet, and StatefulSet: the Agent first cleans up and confirms absence of the old carrier and every old `DeleteOnTaskCleanup` auxiliary object, including the Ray head Service, while retaining shared Ray ConfigMaps. Referenced external PVCs are outside RLark ownership. It then switches the frozen claimed Workload selector and creates the new carrier on a later reconciliation. Ownership conflicts or incomplete evidence block migration. This fallback can interrupt the workload and provides neither cross-Runtime migration nor transactional rollback; it exists to prevent leaked resources and stale claim metadata, not to replace Task replacement. +Upper-layer services should not change `spec.workload.kind` on an already claimed Kubernetes Task as their normal workflow. Replace the Task with a new UID—or use the Job restart/replacement workflow—to change carrier kind. If a direct edit nevertheless reaches the Agent, it invokes a narrow, recoverable compatibility path between Deployment and StatefulSet: the Agent first cleans up and confirms absence of the old carrier and every old `DeleteOnTaskCleanup` auxiliary object, including the Ray head Service, while retaining shared Ray ConfigMaps. Referenced external PVCs are outside RLark ownership. It then switches the frozen claimed Workload selector and creates the new carrier on a later reconciliation. Ownership conflicts or incomplete evidence block migration. This fallback can interrupt the workload and provides neither cross-Runtime migration nor transactional rollback; it exists to prevent leaked resources and stale claim metadata, not to replace Task replacement. -Task deletion, stop, and restart remain pending until every task-owned `DeleteOnTaskCleanup` object is confirmed absent. This includes the primary Deployment, DaemonSet, or StatefulSet and a Ray head Task's UID-owned head Service. Shared Ray bootstrap ConfigMaps are retained; referenced external PVCs are not part of cleanup inventory. A snapshot or deletion failure keeps the Task finalizer so reconciliation can recover safely. +Task deletion, stop, and restart remain pending until every task-owned `DeleteOnTaskCleanup` object is confirmed absent. This includes the primary Deployment or StatefulSet and a Ray head Task's UID-owned head Service. Shared Ray bootstrap ConfigMaps are retained; referenced external PVCs are not part of cleanup inventory. A snapshot or deletion failure keeps the Task finalizer so reconciliation can recover safely. The Ray head Service uses replacement semantics when an RLark-owned selector, port, type, readiness-publication setting, or canonical metadata field drifts. The Agent UID-fences and deletes the current Service, then recreates it on a later reconciliation after absence is observed. This may briefly interrupt access and may allocate different Service addresses or ports. Kubernetes-materialized defaults and allocations, including ClusterIP, IP family fields, NodePort, default protocol and target port, and unrelated metadata, are excluded from the drift projection and do not by themselves trigger replacement. diff --git a/apps/rlark/docs/admin-guide/cluster-delisted-migration.md b/apps/rlark/docs/admin-guide/cluster-delisted-migration.md new file mode 100644 index 0000000..0d632a7 --- /dev/null +++ b/apps/rlark/docs/admin-guide/cluster-delisted-migration.md @@ -0,0 +1,26 @@ +# Migrate Cluster Delist State + +RLark now uses `Cluster.spec.delisted` to represent that an administrator has delisted a cluster. Earlier releases used `Cluster.spec.deleted` for the same durable intent. + +## Upgrade order + +1. Deploy a transition CRD that accepts both `spec.deleted` and `spec.delisted`. +2. Run the migration script against the Kubernetes cluster hosting kcp before removing `spec.deleted` from the CRD: + +```bash +KUBECONFIG=~/.kube/host.kubeconfig \ + ./hack/migrate-cluster-deleted-to-delisted.sh rlark-system kcp-0 +``` + +3. Confirm the dry run reports no remaining resources: + +```bash +KUBECONFIG=~/.kube/host.kubeconfig \ + ./hack/migrate-cluster-deleted-to-delisted.sh rlark-system kcp-0 --dry-run +``` + +4. Deploy the release that removes `spec.deleted` from the Cluster CRD and uses `spec.delisted` exclusively. + +The script obtains the kcp administrator kubeconfig from `rlark-kcp-kubeconfig`, port-forwards the selected kcp Pod, then targets the kcp API rather than the host Kubernetes API. It migrates only Cluster resources where `spec.deleted=true`, sets `spec.delisted=true`, and removes the legacy field with a merge patch. It is idempotent: rerunning it after a successful migration makes no changes. + +Do not apply a CRD that rejects or prunes `spec.deleted` before the migration has run, because the old field may be removed before its meaning is transferred to `spec.delisted`. diff --git a/apps/rlark/docs/api/reference.md b/apps/rlark/docs/api/reference.md index 0e2b757..1e8f263 100644 --- a/apps/rlark/docs/api/reference.md +++ b/apps/rlark/docs/api/reference.md @@ -94,11 +94,11 @@ Image registry credentials use an immutable `ir-<16 lowercase hexadecimal charac | `GET, POST` | `/api/v1/storage/storageclass` | | `PUT, DELETE` | `/api/v1/storage/storageclass/{name}` | | `GET` | `/api/v1/storage/storageclass/provider` | -| `GET` | `/api/v1/storage/storageclass/{name}/{cluster}/list` | -| `POST` | `/api/v1/storage/storageclass/{name}/{cluster}/upload` | -| `GET, DELETE` | `/api/v1/storage/storageclass/{name}/{cluster}/object/{key}` | +| `GET` | `/api/v1/storage/storageclass/{name}/list` | +| `POST` | `/api/v1/storage/storageclass/{name}/upload` | +| `GET, DELETE` | `/api/v1/storage/storageclass/{name}/object/{key}` | -StorageClass routes manage control-plane declarations. Replication and Delivery controllers asynchronously distribute generated rclone CSI resources to the configured target clusters; listing returns managed declarations rather than live Agent queries. +StorageClass routes manage control-plane declarations. Replication and Delivery controllers asynchronously distribute generated rclone CSI resources to the configured target clusters; listing returns managed declarations rather than live Agent queries. File routes access the declared bucket directly. ## Addons diff --git a/apps/rlark/docs/api/swagger.yaml b/apps/rlark/docs/api/swagger.yaml index d67fd99..2be674a 100644 --- a/apps/rlark/docs/api/swagger.yaml +++ b/apps/rlark/docs/api/swagger.yaml @@ -103,6 +103,114 @@ paths: '500': description: Failed to read the Cluster or list Nodes in its agent namespace + /api/v1/clusters/{cluster_id}/cordon: + post: + tags: [Cluster] + summary: Cordon a cluster + description: Administrator-only. Stops new task placement on the cluster. + operationId: cordonCluster + parameters: + - name: cluster_id + in: path + required: true + schema: {type: string} + responses: + '200': {description: Cluster cordoned} + '404': {description: Cluster not found} + + /api/v1/clusters/{cluster_id}/uncordon: + post: + tags: [Cluster] + summary: Uncordon a cluster + description: Administrator-only. Resumes new task placement on the cluster. A delisted or draining cluster must be restored or finish draining first. + operationId: uncordonCluster + parameters: + - name: cluster_id + in: path + required: true + schema: {type: string} + responses: + '200': {description: Cluster uncordoned} + '404': {description: Cluster not found} + '409': {description: Cluster cannot be uncordoned in its current lifecycle state} + + /api/v1/clusters/{cluster_id}/drain: + post: + tags: [Cluster] + summary: Drain a cluster + description: Administrator-only. Stops RLark Tasks in the cluster while the desired draining state remains enabled. + operationId: drainCluster + parameters: + - name: cluster_id + in: path + required: true + schema: {type: string} + responses: + '200': {description: Cluster drain requested} + '404': {description: Cluster not found} + + /api/v1/clusters/{cluster_id}/undrain: + post: + tags: [Cluster] + summary: Cancel cluster drain + description: Administrator-only. Cancels draining and restores the cluster's pre-drain cordon state. + operationId: undrainCluster + parameters: + - name: cluster_id + in: path + required: true + schema: {type: string} + responses: + '200': {description: Cluster drain cancelled} + '404': {description: Cluster not found} + '409': {description: Delisted clusters must be restored before their drain can be cancelled} + + /api/v1/clusters/{cluster_id}/delist: + post: + tags: [Cluster] + summary: Delist a cluster + description: Administrator-only. Stops showing the cluster and its Nodes to users. The cluster can be restored. + operationId: delistCluster + parameters: + - name: cluster_id + in: path + required: true + schema: {type: string} + responses: + '200': {description: Cluster delisted} + '404': {description: Cluster not found} + '409': {description: Cluster must finish draining before it can be delisted} + + /api/v1/clusters/{cluster_id}/delete: + post: + tags: [Cluster] + summary: Delete a cluster + description: Administrator-only. Permanently deletes the Cluster resource. The Cluster controller cleans up owned data-plane resources before final deletion. + operationId: deleteCluster + parameters: + - name: cluster_id + in: path + required: true + schema: {type: string} + responses: + '200': {description: Cluster deletion requested} + '404': {description: Cluster not found} + + /api/v1/clusters/{cluster_id}/restore: + post: + tags: [Cluster] + summary: Restore a cluster + description: Administrator-only. Makes a previously hidden cluster visible again. + operationId: restoreCluster + parameters: + - name: cluster_id + in: path + required: true + schema: {type: string} + responses: + '200': {description: Cluster restored} + '404': {description: Cluster not found} + # ─── Job Collection ─────────────────────────────────────────────── /api/v1/rlinf.io/v1alpha1/jobs: get: diff --git a/apps/rlark/docs/architecture.md b/apps/rlark/docs/architecture.md index 4676abd..bedba62 100644 --- a/apps/rlark/docs/architecture.md +++ b/apps/rlark/docs/architecture.md @@ -122,7 +122,7 @@ graph LR **Pull Controller** (Task example): 1. Watch control plane Task CR create/update/delete events (with finalizer protection) -2. Decode the Task into an execution snapshot and select the Kubernetes Runtime plus the Ray-oriented Kubernetes Workload Adapter; the legacy Deployment, DaemonSet, or StatefulSet discriminator selects its carrier kind +2. Decode the Task into an execution snapshot and select the Kubernetes Runtime plus the Ray-oriented Kubernetes Workload Adapter; the carrier declaration selects Deployment or StatefulSet 3. Compile a declarative object plan whose Adapter selects ownership, lifecycle, owned fields, update mode, and a typed Runtime reconcile profile; ask the Adapter for bounded identity inventory, read that Runtime snapshot, and compute the required operations 4. Execute ordered, idempotent Apply/Delete operations; a `Replace` object is UID-fenced and deleted before a later reconciliation recreates it, and running Tasks are reconciled again when their desired generation changes 5. Apply Workload-owned Pod-template mutations such as Ray bootstrap during compilation, then resolve explicit Environment requirements and bind the cross-domain network sidecar only to the Adapter-authorized PodTemplate target diff --git a/apps/rlark/docs/concepts.md b/apps/rlark/docs/concepts.md index b499616..5852cd2 100644 --- a/apps/rlark/docs/concepts.md +++ b/apps/rlark/docs/concepts.md @@ -176,7 +176,7 @@ A Job contains multiple Task templates, each representing a role in training (e. Before dispatching any Task, the Job Controller validates every template as one desired unit. Template names must be non-empty and unique, and `runtime.kind`, workload kind/version, and carrier type/version must be non-empty open identifiers. For the current Job-to-Ray-on-Kubernetes path, the controller additionally requires a Pod template containing exactly one regular container named `main`. This is an implementation capability check, not a universal Task CRD constraint: the CRD permits omitting `workload.template`, and other Workload Adapter/Runtime combinations define their own requirements. If any template is invalid, the controller creates none of them and reports `Validated=False` on the Job with the failing field path. It does not silently skip individual templates. The Agent independently validates the selected implementation before Runtime effects. The Job Controller does not enforce Ray head count or head replica topology. -The Web UI keeps its existing Job and Workflow creation, editing, cloning, lifecycle, storage, and Worker interactions, but serializes them directly with the generated CRD contracts. New requests use `runtime`, `workload`, `carrier`, `placement`, and `environments`; lifecycle actions write `spec.suspended`; and storage selections become Kubernetes-native Pod template volumes. The UI does not emit the removed `agentType`, nested `kubernetes.workload`, `stopped`, `pvcStorageMap`, or `pvcSizeGbMap` forms. +The Web UI keeps its existing Job and Workflow creation, editing, cloning, lifecycle, storage, and Worker interactions, but serializes them directly with the generated CRD contracts. New requests use `runtime`, `workload`, `carrier`, `placement`, and `environments`; lifecycle actions write `spec.suspended`; and storage selections become Kubernetes-native Pod template volumes. Volume types the form cannot express, such as `emptyDir`, `configMap`, or `secret`, are displayed read-only and preserved from the source definition when a Job is edited or cloned. The UI does not emit the removed `agentType`, nested `kubernetes.workload`, `stopped`, `pvcStorageMap`, or `pvcSizeGbMap` forms. ### Key Properties @@ -364,7 +364,7 @@ status: | Type | Description | Use Case | |------|-------------|----------| -| `Kubernetes` | Run RLark/Ray Tasks using Deployment, DaemonSet, or StatefulSet as the Kubernetes carrier | GPU clusters for large-scale training | +| `Kubernetes` | Run RLark/Ray Tasks using Deployment or StatefulSet as the Kubernetes carrier | GPU clusters for large-scale training | | `Docker` | Planned provider that lowers the common Pod template to Docker containers; not implemented or advertised | Edge devices (robot arm, sensor, camera) | | `Raw` | Planned provider that lowers the common Pod template to host processes; not implemented or advertised | Bare metal servers, embedded devices | @@ -407,7 +407,7 @@ When a Pod is created in the data plane cluster, the Agent's Pod Push controller ```mermaid graph TD job -->|"1:N"| task["Task
(Namespaced: agent-{id})
Exec Unit"] - task -->|"1:1 (K8s workload)"| workload["Deployment /
DaemonSet /
StatefulSet
(Local k8s cluster)"] + task -->|"1:1 (K8s workload)"| workload["Deployment /
StatefulSet
(Local k8s cluster)"] workload -->|"1:N"| pod["Pod + Sidecar
Agent Push reports → Pod CR"] domain["Domain
(Cluster scoped)
Network Isolation"] -->|"1:N (one per cluster)"| dp["DomainPeer
(Namespaced: agent-{id})
Pod Routing Table"] node["Node
(Namespaced)
Compute Node"] diff --git a/apps/rlark/docs/developer-guide/api-conventions.md b/apps/rlark/docs/developer-guide/api-conventions.md index b3d937b..1299bd2 100644 --- a/apps/rlark/docs/developer-guide/api-conventions.md +++ b/apps/rlark/docs/developer-guide/api-conventions.md @@ -462,7 +462,7 @@ the name, so `fieldSelector` on `metadata.name` already expresses it. | `order` | `field [asc\|desc]`; field must match `^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*$`, and an unknown direction is rejected instead of silently sorting ascending. | | `fieldSelector` | Comma-separated `key=value` or `key!=value` pairs; each key must match the same dotted-identifier pattern as `order`. | | `labelSelector` | Comma-separated `key=value` or `key!=value` pairs on the database path, a Kubernetes label selector on the Kubernetes path. Keys must be valid qualified names. | -| `tagSelector` | Comma-separated `key=value` pairs. Applied after the server window. | +| `tagSelector` | Comma-separated `key=value` pairs. Values of the same key are ORed, different keys are ANDed. Applied after the server window. | | `continue` | Opaque Kubernetes continue token. Kubernetes path only. | Selector paths and label keys come from the query string and are therefore diff --git a/apps/rlark/docs/developer-guide/runtime-provider.md b/apps/rlark/docs/developer-guide/runtime-provider.md index 03d23ce..6196a43 100644 --- a/apps/rlark/docs/developer-guide/runtime-provider.md +++ b/apps/rlark/docs/developer-guide/runtime-provider.md @@ -61,8 +61,8 @@ rejected at startup when it is duplicated, its Adapter metadata does not match, or its Runtime kind is not declared by the Adapter. Kubernetes currently registers one `(rlinf.io/ray, v1, Kubernetes)` implementation. The Ray Adapter independently decodes `workload.carrier` and supports the -`rlinf.io/kubernetes-deployment`, `rlinf.io/kubernetes-daemonset`, and -`rlinf.io/kubernetes-statefulset` carriers at version `v1`. These carriers are +`rlinf.io/kubernetes-deployment` and `rlinf.io/kubernetes-statefulset` carriers +at version `v1`. These carriers are not general-purpose native Kubernetes workload implementations. An unclaimed Task is decoded and checked against this exact registry before any @@ -78,7 +78,7 @@ keeping those workflows recoverable after a downgrade or partially migrated update. A complete per-Runtime Bundle/Factory remains a separate assembly step; this document does not claim that it is already present. -Deployment, DaemonSet, and StatefulSet are carrier choices within the single Ray +Deployment and StatefulSet are carrier choices within the single Ray Adapter registration, not Workload selection entries. This compatibility tree is deliberately controller-owned rather than part of @@ -164,7 +164,7 @@ The built-in Ray-on-Kubernetes Workload Adapter owns rendering, desired-object l - Workload-owned template mutations and workload identity, applied once during Workload compilation before rendering the desired Kubernetes objects; cross-domain network injection is instead performed by Environment binding; - image pull secrets, PVCs, and Ray resources; - declarative workload SSH requirements; the Environment layer owns Secret delivery and Pod-template mounting; -- rendering the selected Deployment, DaemonSet, or StatefulSet workload for the Coordinator and Runtime Provider to apply. +- rendering the selected Deployment or StatefulSet workload for the Coordinator and Runtime Provider to apply. The legacy `Operations` adapter remains only as a tested compatibility port; production assembly uses the Ray-on-Kubernetes Adapter and does not call its `ApplyWorkload` method. @@ -180,7 +180,7 @@ Process declarations are a portable-plan contract, are deep-copied, and are incl After the optional user prepare-script boundary and before any Ray stop or start command, `ray-launcher` verifies that the configured Ray CLI can be resolved and is executable. A missing or non-executable CLI fails the primary. On the head, the embedded Ray SDK cluster check remains non-fatal for compatibility, but the following `ray status` command is fatal. On a worker, failure to start `ray start --block` and any later non-zero Ray exit both fail the primary; an ordinary process exit preserves the Ray exit code through the launcher and supervisor. -Workload semantics are selected by `spec.workload.kind` and `spec.workload.version`; arbitrary JSON `spec.workload.parameters` is strictly decoded by the selected Adapter. `spec.workload.carrier` independently contains the Runtime-specific carrier `type`, `version`, and arbitrary JSON `parameters`. Deployment and StatefulSet carriers accept only `{replicas}`; the DaemonSet carrier accepts only an empty object. Environment declarations use the same type/version/parameters envelope in an ordered list and may repeat. Each selected Environment resolver owns any uniqueness rule for its contract. Unknown types, versions, malformed parameters, and unsupported Runtime capabilities fail before Provider effects. +Workload semantics are selected by `spec.workload.kind` and `spec.workload.version`; arbitrary JSON `spec.workload.parameters` is strictly decoded by the selected Adapter. `spec.workload.carrier` independently contains the Runtime-specific carrier `type`, `version`, and arbitrary JSON `parameters`. Deployment and StatefulSet carriers accept only `{replicas}`. Environment declarations use the same type/version/parameters envelope in an ordered list and may repeat. Each selected Environment resolver owns any uniqueness rule for its contract. Unknown types, versions, malformed parameters, and unsupported Runtime capabilities fail before Provider effects. The Task decoder maps every Environment envelope generically without enumerating built-in types: it checks only common required fields, preserves ordering and duplicates, deep-copies parameters, and leaves type-specific uniqueness, version, and parameter validation to the selected Runtime resolver. Kubernetes Environment resolvers are installed through an immutable type-to-resolver registry; empty types, nil resolvers, and duplicate type registrations fail during construction. The composed resolver groups declarations by type in first-appearance order and passes every declaration of one type to that type's resolver in original order. Network, SSH, and TensorBoard currently reject multiple declarations themselves; future Environment contracts may register a resolver that accepts and resolves multiple declarations. Unknown types decode successfully but fail closed during resolution before Provider observation or effects. The portable execution plan preserves this same Environment envelope instead of splitting requirements into network, secret, tool, service, storage, device, or config fields. Each plan requirement contains `name`, open-string `type` and `version`, copied parameters, and its consumer selectors. SSH authorized public keys are retained only in the local execution declaration and omitted from portable requirement parameters and workload transport; the SSH handler acquires them through the material boundary. @@ -275,13 +275,13 @@ Runtime `ListCandidateObjects`, legacy `ListManaged`, and `ListInstances` are at The selected Workload Adapter derives a bounded inventory from the frozen Task and implementation identity. Candidates are declared in creation order. Identity cleanup uses `LifecycleDeleteOnTaskCleanup` as the deletion authorization and removes the last currently present exact-name cleanup candidate per reconciliation, reversing that order; it does not maintain a Kubernetes-kind allow-list. The Ray-on-Kubernetes Adapter declares the SSH Secret before its workload consumers so workload objects disappear before the Secret. It does not declare PVC candidates: referenced external PVCs are outside RLark ownership, and generic ephemeral PVCs follow their Pod lifecycle. The Kubernetes Provider executes only Adapter declarations; it does not compute Ray names or infer Ray roles. Candidate names and labels locate objects but never prove ownership: the Provider verifies canonical Task identity and object UID where applicable before returning an object. Query failure fails the complete snapshot atomically. Instance observation UID-fences the workload root, lists only Pods selected by that workload, and queries Pod Events with the `involvedObject.uid` field selector. -Each desired Kubernetes object selects a typed reconcile profile instead of asking the Provider to infer update policy from its manifest kind. The current Ray Adapter selects workload projection for Deployment, DaemonSet, and StatefulSet carriers, Service projection for the head Service, and shared-field projection for bootstrap ConfigMaps. +Each desired Kubernetes object selects a typed reconcile profile instead of asking the Provider to infer update policy from its manifest kind. The current Ray Adapter selects workload projection for Deployment and StatefulSet carriers, Service projection for the head Service, and shared-field projection for bootstrap ConfigMaps. The immutable Runtime-owned registry dispatches the explicitly selected versioned executor and verifies that the typed manifest matches that profile. A missing, unknown, duplicate, or mismatched profile is rejected. Each executor implements validation, comparison, and update preparation for one bounded Kubernetes mechanism, such as comparing a Service-owned projection. The Adapter remains responsible for selecting the mechanism that represents its Workload contract. Before scheduling a Service replacement, the Coordinator compares only the Service profile's owned fields through this registry. API-server materialization and allocation—including ClusterIP, ClusterIPs, IP family settings, NodePort, default protocol and target port, traffic policy, session affinity, and unrelated metadata—does not cause replacement drift. The built-in registry remains intentionally narrow; its workload executor is not a general CRD update policy. -Before comparing a desired Deployment, DaemonSet, or StatefulSet Pod template with the object returned by the API server, the Kubernetes provider conservatively preserves commonly materialized Pod and container defaults where the desired template left those fields unspecified. The current compatibility set covers Pod restart policy, DNS policy, scheduler name, termination grace period, an empty Pod security context, and container image-pull and termination-message defaults. It prevents common repeated updates without overriding explicit desired values, but it does not reproduce every version-dependent nested container, probe, volume, or ephemeral-container default. The provider currently treats the complete Pod template as Agent-managed; fields injected into that template by mutating admission webhooks or another controller are not a supported ownership boundary and may be overwritten and re-injected. Clusters should not apply such mutation to RLark-managed workload templates until an explicit Agent-owned field projection or server-side-apply ownership contract is implemented. +Before comparing a desired Deployment or StatefulSet Pod template with the object returned by the API server, the Kubernetes provider conservatively preserves commonly materialized Pod and container defaults where the desired template left those fields unspecified. The current compatibility set covers Pod restart policy, DNS policy, scheduler name, termination grace period, an empty Pod security context, and container image-pull and termination-message defaults. It prevents common repeated updates without overriding explicit desired values, but it does not reproduce every version-dependent nested container, probe, volume, or ephemeral-container default. The provider currently treats the complete Pod template as Agent-managed; fields injected into that template by mutating admission webhooks or another controller are not a supported ownership boundary and may be overwritten and re-injected. Clusters should not apply such mutation to RLark-managed workload templates until an explicit Agent-owned field projection or server-side-apply ownership contract is implemented. Ray bootstrap ConfigMaps use RLark-reserved fixed names and are retained across Task cleanup. The Adapter explicitly projects ownership of their `data`, `binaryData`, and Ray-script identity annotation into each desired object; the Provider updates only that projection and does not recognize Ray names. Equal script content is a zero-write steady state, while drift is repaired in place. Other labels and annotations are preserved. A legacy fixed-name ConfigMap without the identity annotation can be normalized, but conflicting Ray-role metadata is left untouched and reported as a conflict. A shared object with a missing or unsupported projection is rejected rather than updated as a whole. Operators must not assign these reserved names to unrelated ConfigMaps or let another manager write the script fields. @@ -334,7 +334,7 @@ The Kubernetes execution path coalesces desired-state, Runtime-state, and period A shared cluster-Agent resync starts immediately and scans management Tasks assigned or eligible for the same Host with pagination and a bounded 10,000-Task budget. Host claim and eligibility rules are shared with reconciliation rather than reimplemented by the scanner: an effective claim requires the Agent finalizer, and a claimed Task remains assigned to its owning Host despite selector drift or deletion. Subsequent scans use a jittered interval in the range 80–120% of `--task-runtime-resync-period`; the default is `15m`, `0` disables the scan, and a negative value is rejected at startup. A transient management API list failure is logged and retried after the next interval instead of terminating the Agent. Trigger-bridge pressure is summarized after the round and does not stop later Tasks or pages from being visited. The scan emits periodic full-reconciliation hints and does not treat its in-memory progress as desired state. -For a proven-owned primary Deployment, DaemonSet, or StatefulSet, normal Apply writes canonical root annotations for Runtime identity (ID/kind), Workload identity (type/version), compiled and executable digests, and the non-secret binding revision. Existing Task name, namespace, UID, generation, and domain annotations remain the ownership and lifecycle compatibility contract. The Job restart annotation is not copied to Runtime objects and is not interpreted by the Agent. Metadata is written only on the root object's top-level annotations, not its Pod template, so metadata repair alone does not roll out Pods. Missing, malformed, or mismatched canonical revision metadata cannot authorize targeted observation and selects full reconciliation. For a canonical-owned workload, full Apply repairs this complete canonical execution revision only after Task ownership (including the Task UID annotation), the current object instance from the UID-fenced reference, and selector compatibility have been proved. A narrowly compatible legacy workload may instead be adopted from its name, namespace, kind, and selector shape when no conflicting ownership metadata exists; that adoption writes canonical Task identity and revision metadata but preserves the Pod template. Subsequent canonical-owned Apply can repair the revision at the same Task generation, making identifier normalization and digest-algorithm upgrades recoverable. Execution revision annotations are ordering and optimization evidence, not ownership proof by themselves. +For a proven-owned primary Deployment or StatefulSet, normal Apply writes canonical root annotations for Runtime identity (ID/kind), Workload identity (type/version), compiled and executable digests, and the non-secret binding revision. Existing Task name, namespace, UID, generation, and domain annotations remain the ownership and lifecycle compatibility contract. The Job restart annotation is not copied to Runtime objects and is not interpreted by the Agent. Metadata is written only on the root object's top-level annotations, not its Pod template, so metadata repair alone does not roll out Pods. Missing, malformed, or mismatched canonical revision metadata cannot authorize targeted observation and selects full reconciliation. For a canonical-owned workload, full Apply repairs this complete canonical execution revision only after Task ownership (including the Task UID annotation), the current object instance from the UID-fenced reference, and selector compatibility have been proved. A narrowly compatible legacy workload may instead be adopted from its name, namespace, kind, and selector shape when no conflicting ownership metadata exists; that adoption writes canonical Task identity and revision metadata but preserves the Pod template. Subsequent canonical-owned Apply can repair the revision at the same Task generation, making identifier normalization and digest-algorithm upgrades recoverable. Execution revision annotations are ordering and optimization evidence, not ownership proof by themselves. The binding revision is a canonical `sha256:` digest of binding kind/name, trusted profile identity and revision, sorted concrete target consumers, ownership, lifecycle, source revision, handler algorithm, and value type. It excludes raw binding values and Secret contents. The executable hash likewise replaces Kubernetes Secret `data` and `stringData` with a fixed marker before hashing; the real desired Secret remains available to the Runtime comparator so rotation still produces an in-place Secret update. `ResolvedBinding.Value` may contain only controlled, serializable, non-sensitive options; raw keys, tokens, and credentials require a separate restricted in-process material mechanism and must not enter portable Plans, transport, Status, diagnostics, errors, hashes, or revisions. Production diagnostic sanitization cannot be disabled at runtime; observer failures or panics do not change reconciliation, Status, or Runtime effects. Before wiring new diagnostic payload shapes, extend and test sanitization for every value-bearing field: the current generic sanitizer covers environment `value` fields, sensitive-key names, and Secret `data`/`stringData`, but does not infer secrets embedded in container `command`/`args` or in non-sensitive-key `TypedOptions` values. Environment-based Network, workload SSH, and TensorBoard mutation plus Runtime-mandatory RLark tools and supervisor assembly are used in production. @@ -349,7 +349,6 @@ Normal pull reconciliation does not stop merely because a Task is already `Runni | Task claim and finalizer | Supported | Not implemented | Not implemented | | Deployment | Supported | Not applicable | Not applicable | | StatefulSet | Supported | Not applicable | Not applicable | -| DaemonSet | Supported | Not applicable | Not applicable | | Container/process execution | Kubernetes Pods | Spec scaffolding only | Spec scaffolding only | | Status observation | Supported | Not implemented | Not implemented | | Runtime logs | Kubernetes Pod fallback | Not implemented | Not implemented | diff --git a/apps/rlark/docs/examples/update-kcp-crds.sh b/apps/rlark/docs/examples/update-kcp-crds.sh new file mode 100755 index 0000000..0ae716d --- /dev/null +++ b/apps/rlark/docs/examples/update-kcp-crds.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Updates RLark CRDs in the kcp API hosted by the Kubernetes cluster in KUBECONFIG. +# Usage: KUBECONFIG=/path/to/host.kubeconfig ./update-kcp-crds.sh [kcp-namespace] [kcp-pod] +: "${KUBECONFIG:?KUBECONFIG must point to the host Kubernetes kubeconfig}" +HOST_KUBECONFIG="$KUBECONFIG" +KCP_NAMESPACE="${1:-rlark-system}" +KCP_POD="${2:-kcp-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" +KCP_KUBECONFIG="$(mktemp)" +CRD_DIR="$(mktemp -d)" +PORT_FORWARD_LOG="$(mktemp)" +PORT_FORWARD_PID="" +KCP_PORT="" + +cleanup() { + if [[ -n "$PORT_FORWARD_PID" ]]; then + kill "$PORT_FORWARD_PID" 2>/dev/null || true + wait "$PORT_FORWARD_PID" 2>/dev/null || true + fi + rm -f "$KCP_KUBECONFIG" "$PORT_FORWARD_LOG" + rm -rf "$CRD_DIR" +} +trap cleanup EXIT + +kubectl --kubeconfig "$HOST_KUBECONFIG" -n "$KCP_NAMESPACE" \ + get configmap rlark-kcp-kubeconfig \ + -o go-template='{{index .data "admin.kubeconfig"}}' > "$KCP_KUBECONFIG" + +KCP_PORT="$(python3 - <<'PY' +import socket +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +)" + +kubectl --kubeconfig "$HOST_KUBECONFIG" -n "$KCP_NAMESPACE" \ + port-forward "pod/$KCP_POD" "$KCP_PORT:6443" >"$PORT_FORWARD_LOG" 2>&1 & +PORT_FORWARD_PID=$! + +for _ in {1..20}; do + if grep -q "Forwarding from 127.0.0.1:$KCP_PORT" "$PORT_FORWARD_LOG"; then + break + fi + if ! kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then + cat "$PORT_FORWARD_LOG" >&2 + exit 1 + fi + sleep 1 +done + +if ! grep -q "Forwarding from 127.0.0.1:$KCP_PORT" "$PORT_FORWARD_LOG"; then + cat "$PORT_FORWARD_LOG" >&2 + exit 1 +fi + +KCP_CONTEXT="$(kubectl --kubeconfig "$KCP_KUBECONFIG" config current-context)" +KCP_SERVER="$(kubectl --kubeconfig "$KCP_KUBECONFIG" config view --raw --minify -o jsonpath='{.clusters[0].cluster.server}')" +KCP_CLUSTER="$(kubectl --kubeconfig "$KCP_KUBECONFIG" config view --raw --minify -o jsonpath='{.contexts[0].context.cluster}')" +KCP_SERVER_HOSTPORT="${KCP_SERVER#https://}" +KCP_SERVER_HOST="${KCP_SERVER_HOSTPORT%%:*}" +KCP_SERVER_PATH="${KCP_SERVER_HOSTPORT#*/}" +if [[ "$KCP_SERVER_PATH" == "$KCP_SERVER_HOSTPORT" ]]; then + KCP_SERVER_PATH="" +else + KCP_SERVER_PATH="/$KCP_SERVER_PATH" +fi +kubectl --kubeconfig "$KCP_KUBECONFIG" config set-cluster "$KCP_CLUSTER" \ + --server="https://127.0.0.1:$KCP_PORT$KCP_SERVER_PATH" \ + --tls-server-name="$KCP_SERVER_HOST" >/dev/null + +make -C "$PROJECT_ROOT/api" controller-gen +CONTROLLER_GEN="$(make -s -C "$PROJECT_ROOT/api" print-controller-gen)" +"$CONTROLLER_GEN" crd:maxDescLen=0,allowDangerousTypes=true \ + paths="$PROJECT_ROOT/api/rlark.io/..." \ + output:crd:artifacts:config="$CRD_DIR" + +kubectl --kubeconfig "$KCP_KUBECONFIG" --context "$KCP_CONTEXT" \ + apply --validate=false -f "$CRD_DIR" + +kubectl --kubeconfig "$KCP_KUBECONFIG" --context "$KCP_CONTEXT" \ + get crd clusters.rlinf.io \ + -o jsonpath='{.spec.versions[?(@.storage==true)].schema.openAPIV3Schema.properties.spec.properties}' +echo diff --git a/apps/rlark/docs/reference/configuration.md b/apps/rlark/docs/reference/configuration.md index 40fdc78..5d6b388 100644 --- a/apps/rlark/docs/reference/configuration.md +++ b/apps/rlark/docs/reference/configuration.md @@ -163,7 +163,6 @@ Data plane agent. Deployed on each cluster or node. Manages node registration, T | `--task-runtime-resync-period` | duration | `15m` | Base period for cluster-mode shared Task Runtime repair scans; intervals use 80–120% jitter, `0` disables scans, and negative values are invalid; node-only mode accepts the shared flag but does not run the scanner | | `--addon-pull-controller-workers` | int | `8` | Maximum concurrent Addon pull reconciles | | `--task-deployment-push-controller-workers` | int | `8` | Maximum concurrent Task Deployment push reconciles | -| `--task-daemonset-push-controller-workers` | int | `8` | Maximum concurrent Task DaemonSet push reconciles | | `--task-statefulset-push-controller-workers` | int | `8` | Maximum concurrent Task StatefulSet push reconciles | | `--node-push-controller-workers` | int | `8` | Maximum concurrent Node push reconciles | | `--pod-push-controller-workers` | int | `8` | Maximum concurrent Pod push reconciles | diff --git a/apps/rlark/docs/reference/crd.md b/apps/rlark/docs/reference/crd.md index 01a0c06..4a4035b 100644 --- a/apps/rlark/docs/reference/crd.md +++ b/apps/rlark/docs/reference/crd.md @@ -381,6 +381,8 @@ Responses: - `spec`: `object`, optional - ClusterSpec declares the durable operational state of a data-plane cluster. - `agentNamespace`: `string`, required - AgentNamespace is the management-plane namespace that contains the agent heartbeat Lease and Node CRs. - `cordoned`: `boolean`, optional + - `delisted`: `boolean`, optional + - `drainPreviousCordoned`: `boolean`, optional - `draining`: `boolean`, optional - `status`: `object`, optional - ClusterStatus reports the controller's current observation of a data-plane cluster. - `conditions`: `array`, optional @@ -393,7 +395,7 @@ Responses: - `type`: `string`, required - type of condition in CamelCase or in foo.example.com/CamelCase. - `lastHeartbeatTime`: `string`, optional - `onlineNodes`: `integer`, optional - - `phase`: `string`, optional, enum=Online,Offline,Downgraded,Cordoned,Draining - ClusterPhase summarizes the current lifecycle state of a data-plane cluster. The set is closed in the CRD schema: add... + - `phase`: `string`, optional, enum=Online,Offline,Downgraded,Cordoned,Draining,Delisted - ClusterPhase summarizes the current lifecycle state of a data-plane cluster. The set is closed in the CRD schema: add... - `totalNodes`: `integer`, optional ## Domain diff --git a/apps/rlark/docs/storage-api.md b/apps/rlark/docs/storage-api.md index ea01279..cddc89e 100644 --- a/apps/rlark/docs/storage-api.md +++ b/apps/rlark/docs/storage-api.md @@ -20,6 +20,7 @@ Lists StorageClass declarations managed by RLark from the management plane. It d "data": { "ceph-rbd": { "name": "ceph-rbd", + "storageClassName": "sc-0123456789abcdef", "clusters": ["agent-beijing", "agent-shanghai"], "description": "Ceph RBD StorageClass", "bucket": "", @@ -36,7 +37,8 @@ Lists StorageClass declarations managed by RLark from the management plane. It d | Field | Type | Description | |------|------|------| -| `name` | `string` | StorageClass name | +| `name` | `string` | Human-readable StorageClass declaration name | +| `storageClassName` | `string` | Generated `sc-` Kubernetes StorageClass name deployed to target data-plane clusters; use this value in PVC `spec.storageClassName` | | `clusters` | `[]string` | Configured target cluster IDs for this StorageClass | | `description` | `string` | Human-readable description | | `bucket` | `string` | Bucket name | @@ -44,20 +46,20 @@ Lists StorageClass declarations managed by RLark from the management plane. It d | `endpoint` | `string` | Storage service endpoint | | `region` | `string` | Storage region | | `pathStyle` | `bool` | Whether path-style addressing is used | -| `accessKeyId` | `string` | Access Key ID; the Access Key Secret is never returned | +| `accessKeyId` | `string` | Masked Access Key ID (`*****`); the Access Key Secret is never returned | ### 2. List Storage Providers **GET** `/api/v1/storage/storageclass/provider` -Lists the supported storage providers, including 31 providers such as AWS S3, Alibaba Cloud OSS, MinIO, and Ceph. +Lists the supported storage providers. Any authenticated user can read this catalog and create, update, or delete StorageClasses. The catalog is the single source of truth for the `provider` values accepted when creating or updating a StorageClass: submit `value`, and use `name` for display only. rclone does not reject an unknown provider — it logs a warning and falls back to its generic profile — so the Gateway rejects any `provider` that is not in this catalog. #### Example Response ```json { "data": [ { "name": "AWS S3", "value": "AWS" }, - { "name": "Alibaba Cloud OSS", "value": "Alibaba" }, - { "name": "MinIO", "value": "MinIO" } + { "name": "阿里云 OSS", "value": "Alibaba" }, + { "name": "MinIO", "value": "Minio" } ], "success": true } @@ -68,10 +70,32 @@ Lists the supported storage providers, including 31 providers such as AWS S3, Al Creates a StorageClass declaration in the management plane. Replication and Delivery controllers asynchronously distribute the generated rclone CSI StorageClass and its Secret to the requested target clusters. +The Gateway normalizes and validates the S3 configuration before storing it: + +- `provider` must match a catalog entry. The canonical value, the display name, and the legacy spellings written by older clients are all accepted, and the record is always stored as the canonical value. An unrecognized value is rejected with `400`. +- `endpoint` must be a bucket-free `http`/`https` URL; a missing scheme defaults to `https`. The bucket name is never embedded in the endpoint because both the Gateway S3 client and rclone compose virtual-hosted URLs themselves. +- `endpoint` may be left empty only for providers that publish a regional endpoint format, such as Alibaba Cloud OSS, in which case `region` is required and the endpoint is derived from it. +- `path_style` is forced to `false` for providers that only answer virtual-hosted requests, such as Alibaba Cloud OSS, Tencent COS, and Huawei OBS. Otherwise the submitted value is kept. +- After structural validation, the Gateway performs an authenticated `HeadBucket` request with a bounded timeout. Create and update requests are rejected with `400 INVALID_REQUEST` when the configured credentials cannot access the bucket. The response does not include provider error details or credentials. + +Existing records written before this validation existed are canonicalized on read, so the Gateway S3 client and the rclone config delivered to the data plane always agree on the provider and the addressing style. + +Validation failures use stable machine-readable error codes and non-sensitive structured details. In particular, omitted required fields return `400` with code `STORAGE_CLASS_REQUIRED_FIELDS` and `details.fields`, for example: + +```json +{ + "code": "STORAGE_CLASS_REQUIRED_FIELDS", + "message": "bucket are required", + "details": { "fields": ["bucket"] } +} +``` + +Clients must use `code` and `details.fields` for localized field feedback instead of translating the human-readable `message`. + ### 4. Update a StorageClass **PUT** `/api/v1/storage/storageclass/{name}` -Updates the object storage configuration and target clusters for the specified StorageClass declaration. The `clusters` field is the desired final target-cluster set; the distribution workflow cleans up resources from clusters omitted from that set. During an update, `access_key_secret` may be left empty, and the Gateway reuses the existing key. +Updates the object storage configuration and target clusters for the specified StorageClass declaration. The `clusters` field is the desired final target-cluster set; the distribution workflow cleans up resources from clusters omitted from that set. During an update, `access_key_id` and `access_key_secret` may be left empty; the Gateway reads and reuses the corresponding stored values, so credentials are not replaced with empty values. A listed `accessKeyId` is masked with five asterisks; clients must treat any submitted `access_key_id` containing `*****` as unchanged rather than as a replacement. When credentials are retained, the Gateway validates the updated connection with the previously stored Access Key ID and secret. ### 5. Delete a StorageClass **DELETE** `/api/v1/storage/storageclass/{name}` @@ -79,13 +103,21 @@ Updates the object storage configuration and target clusters for the specified S Deletes the StorageClass declaration. Distribution finalizers asynchronously remove its Delivery declarations and generated StorageClass and Secret from all associated clusters. ### 6. List Bucket Files -**GET** `/api/v1/storage/storageclass/{name}/{cluster}/list` +**GET** `/api/v1/storage/storageclass/{name}/list` -Lists files in the StorageClass bucket in the specified cluster. +Lists files in the StorageClass bucket. The Gateway reads the bucket directly from the control-plane StorageClass declaration. #### Path Parameters - `name`: StorageClass name -- `cluster`: Cluster ID, such as `agent-beijing` + +#### Query Parameters +- `prefix` (optional): Object-key prefix to list. A non-root directory prefix is slash-separated and ends with `/`, for example `a/b/c/`. +- `delimiter` (optional): Object-key delimiter. Defaults to `/`; with this delimiter, immediate child directories are returned in `common_prefixes`. +- `maxKeys` (optional): Maximum number of objects and common prefixes to return. + +#### Directory Navigation + +Object storage has prefixes rather than real directories. The file browser treats a non-empty directory prefix as a canonical slash-terminated value. For example, the directory path `a/b/c` is represented as `a/b/c/`; its parent is `a/b/`, and breadcrumbs target `a/`, `a/b/`, and `a/b/c/`. This navigation remains available when a prefix contains no direct objects or child prefixes. #### Example Response ```json @@ -99,13 +131,12 @@ Lists files in the StorageClass bucket in the specified cluster. ``` ### 7. Upload a File -**POST** `/api/v1/storage/storageclass/{name}/{cluster}/upload` +**POST** `/api/v1/storage/storageclass/{name}/upload` Uploads a file to the specified StorageClass bucket using multipart/form-data. #### Path Parameters - `name`: StorageClass name -- `cluster`: Cluster ID #### Request Body Use multipart/form-data with the uploaded file in the `file` field. @@ -119,13 +150,12 @@ Use multipart/form-data with the uploaded file in the `file` field. ``` ### 8. Download a File -**GET** `/api/v1/storage/storageclass/{name}/{cluster}/object/*key` +**GET** `/api/v1/storage/storageclass/{name}/object/*key` Downloads an object from the specified bucket and returns the raw file content. #### Path Parameters - `name`: StorageClass name -- `cluster`: Cluster ID - `key`: Object path, such as `model-checkpoint.pt` or `logs/training.log` #### Response @@ -133,13 +163,12 @@ Downloads an object from the specified bucket and returns the raw file content. - `404`: File not found ### 9. Delete a File -**DELETE** `/api/v1/storage/storageclass/{name}/{cluster}/object/*key` +**DELETE** `/api/v1/storage/storageclass/{name}/object/*key` Deletes an object from the specified bucket. #### Path Parameters - `name`: StorageClass name -- `cluster`: Cluster ID - `key`: Object path #### Example Response diff --git a/apps/rlark/docs/user-guide/clusters-nodes.md b/apps/rlark/docs/user-guide/clusters-nodes.md index 326435d..9835351 100644 --- a/apps/rlark/docs/user-guide/clusters-nodes.md +++ b/apps/rlark/docs/user-guide/clusters-nodes.md @@ -9,13 +9,22 @@ Use this guide to find a data-plane cluster and confirm that it has suitable Wor 3. Check the cluster status, online rate, and Worker count. 4. Open the cluster that you plan to use. -A Cluster is a cluster-scoped RLark resource. Its cluster Agent renews the control-plane `heartbeat` Lease every 15 seconds, and the Lease expires after 45 seconds. Every registered Worker also requires its own `node-heartbeat-` Lease from `rlark-agent-node`; a missing or expired node Lease makes only that Worker offline. The current status has the following meanings: +A Cluster is a cluster-scoped RLark resource. `spec.agentNamespace` is the authoritative membership boundary: the cluster details and node count include only Node CRs in that namespace. `rlark.io/cluster-id` is a compatibility lookup label and does not establish membership. Its cluster Agent renews the control-plane `heartbeat` Lease every 15 seconds, and the Lease expires after 45 seconds. Every registered Worker also requires its own `node-heartbeat-` Lease from `rlark-agent-node`; a missing or expired node Lease makes only that Worker offline. + +The **Status** column and status filter use the following cluster phases: - **Online**: the cluster heartbeat is current and all registered Workers are online. - **Offline**: the heartbeat Lease is missing or expired. - **Downgraded**: the cluster heartbeat is current, but one or more registered Workers are offline. -- **Cordoned**: scheduling is disabled by durable Cluster intent. -- **Draining**: the Cluster is draining as a durable desired state. +- **Cordoned**: scheduling is disabled by durable Cluster intent. Existing Tasks are not stopped by this action. +- **Draining**: administrators have requested an orderly drain. RLark treats already Succeeded or Failed Tasks as drained, records a drain stop request for each remaining Task, and waits until each reports the stopped state for its current generation. The owning Job controller applies the suspension for Job-owned Tasks; the Cluster controller directly suspends standalone Tasks. Drain checks use exponential backoff from 1 second to 1 minute. A drain that remains incomplete for 30 minutes retains the requested state and reports a `Progressing=True`, `Reason=DrainTimedOut` condition listing the blocked Tasks for recovery. Completing a drain does not by itself change the durable drain or cordon intent. After the current-generation `Progressing=False`, `Reason=DrainComplete` condition appears, use Delist to hide the cluster, Undrain to restore the cordon state saved when the drain began, or Uncordon to cancel the drain intent and resume scheduling. Cancelling a drain clears the stop requests and resumes only the standalone Tasks the drain suspended; a Task that was already suspended before the drain keeps its own desired state. +- **Delisted**: administrators have hidden the cluster and its Nodes from the user console. The Nodes page and Node list API omit those Nodes for non-administrators; restoring the cluster makes them visible again. Administrators can still inspect the cluster and its Nodes to operate or restore it. + +The list also displays total, online, and offline Worker counts and calculates the online rate as `online Workers / total Workers`. The count and rate describe Worker heartbeat observation; they do not change the cluster phase or scheduling eligibility on their own. + +Administrators can use **Cordon** to pause new scheduling, **Uncordon** to resume new scheduling after a cluster is cordoned or after its drain has completed, **Drain** to stop only RLark Tasks in the selected cluster, **Delist** to stop showing it to users after a drain has completed, and **Restore** to make a delisted cluster visible again without changing its prior scheduling intent. Delisting is distinct from the **Offline** heartbeat phase. A draining cluster can be Delisted or Uncordoned only after its current-generation `Progressing=False`, `Reason=DrainComplete` condition confirms completion; while the drain is incomplete, these actions are rejected. Uncordoning a fully drained cluster also clears `spec.draining` and the saved cordon state, so the cluster becomes schedulable again instead of staying blocked by the drain intent. Uncordon does not repair agent connectivity or node health; after it clears the scheduling intent, the cluster phase reflects its observed heartbeat and node state. A delisted cluster must be restored before it can be uncordoned. Job outcomes continue to be aggregated from the observed Task states. + +When creating a Job, the console offers **Online** and **Downgraded** clusters. The Job controller also enforces this server-side when it resolves a Task's node selector: it accepts only online, schedulable Nodes in exactly one Cluster whose `spec.cordoned`, `spec.draining`, and `spec.delisted` are all false and whose phase is **Online** or **Downgraded**. Cordoned, Draining, Offline, and Delisted clusters are not candidates, including for Jobs created through the API. A Downgraded cluster can schedule only onto its online, schedulable Workers; its offline Workers remain ineligible. The Gateway rejects new standalone Task requests targeting a Cluster agent namespace while that Cluster is Cordoned, Draining, or Delisted, but it permits updates and patches to existing Tasks for operational recovery. Existing Tasks are not stopped by Cordon; Drain remains responsible for stopping existing RLark Tasks. > **Screenshot note:** Screenshots are from an example environment. Resource names and data are illustrative; your environment will differ. diff --git a/apps/rlark/docs/user-guide/console.md b/apps/rlark/docs/user-guide/console.md index 05f3ad6..2258665 100644 --- a/apps/rlark/docs/user-guide/console.md +++ b/apps/rlark/docs/user-guide/console.md @@ -25,7 +25,7 @@ RLark has two login entry points: | Page | Purpose | | -------- | -------------------------------------------------------------- | -| Overview | Dashboard summary of clusters, nodes, robots, and running jobs | +| Overview | Dashboard summary of clusters, schedulable workload nodes, robots, and running jobs | | Clusters | Browse and inspect onboarded data-plane clusters | | Nodes | Filter and inspect node resources, scheduling, and health | | Jobs | Create, monitor, and manage training jobs | @@ -56,6 +56,36 @@ The **Image Registries** page stores private-registry credentials and selects th Use **Refresh** to request the latest data without leaving the current page. While the request is in progress, the affected table or dashboard is dimmed, its controls are temporarily disabled, and a centered progress indicator is shown. Existing rows remain visible until the refreshed response arrives. This behavior is used consistently across overview dashboards, clusters and nodes, domains, jobs and Workers, storage and files, SSH keys, image registries, and system configuration. +### Interface Language + +The language switch in the header applies to both Chinese and English. Besides static labels, it also localizes values that the console derives from control-plane state: node status reasons, Addon phases, the CPU core unit, derived node kinds in Worker filters, and list separators. The login page and the WebTerminal page follow the same switch. Values passed through verbatim from the data plane — node labels, operating system and architecture identifiers, and other server-provided strings — are shown as reported and are not translated. + +### Long Values in Node Tables + +In the node detail Worker table, long instance names are truncated with an ellipsis, and in the expanded Pod sub-table the network domain is truncated the same way. Hover over a truncated value to see the full text in a tooltip. This keeps the status column visible without horizontal scrolling. + +### Filtering Jobs by Tag + +The tag filter in the job list combines the selection the same way the Gateway does: different tag keys are ANDed, and several values of the same key are ORed. A job is listed only when it carries every selected key, and for each of those keys at least one of the selected values. Selecting keys that no single job carries together therefore returns an empty list; this is the filter semantics, not a failed request. The popover states the rule above the columns, and the **All / None** shortcut applies to the values of the key currently in focus, never to the key list. + +A key comes into focus when you click its row, and the value column keeps showing that key until you click another one. Hovering does not change the focus, so moving the pointer towards the values never replaces the list. The checkbox in front of a key is a separate control: it selects or clears all values of that key at once. + +### Role Count in the Job List + +The **Roles** column counts the distinct role names of the Job, the same names the job detail page and the job editor show. It is not derived from `Task.spec.role`, which is only a coarse grouping hint (`Actor`, `Rollout`, `Env`) and collapses several business roles into one value. + +### Copy Buttons + +Copy buttons report success inline instead of silently succeeding: the icon switches to a check mark for about 1.6 seconds and then returns to the copy icon. The image copy button in the job detail role runtime panel also turns green with a short pulse and changes its tooltip to **Copied**. A button that stays unchanged means the copy did not happen, usually because the browser blocked clipboard access. + +Every copy button in the console shares one clipboard path. Browsers expose the async clipboard API only in a secure context (HTTPS or `localhost`), so when the console is served over plain HTTP the copy automatically falls back to a compatible path instead of failing. This covers the SSH key, SSH connect command, Worker SSH command, API example, deployment YAML, workflow YAML, and job log copy buttons. Success feedback appears only after the copy actually succeeds. + +### Closing Form Dialogs + +Dialogs that collect input — creating, cloning, editing, or restart-editing a job, creating a workflow, creating or editing a storage class, adding or editing image registry credentials, adding an SSH key, creating a domain, and editing job tags — do not close when you click the dimmed area outside the dialog. Because the console cannot restore discarded input, these dialogs close only through their close button in the upper-right corner or their **Cancel** button; the job, workflow, storage class, and image registry dialogs also close with `Esc`. + +Confirmation dialogs that collect no input, such as job lifecycle actions, job deletion, the restart mode choice, and cluster lifecycle actions, still close when you click outside them. + ## API Equivalent Supported resource operations are available through the [Gateway API](../api/reference.md). The standalone Gateway defaults to `http://:8080`; an `rlarkadm` deployment uses its configured Service and UI proxy. The API reference is authoritative because not every UI interaction has a one-to-one public endpoint. diff --git a/apps/rlark/docs/user-guide/jobs.md b/apps/rlark/docs/user-guide/jobs.md index 923482c..d2918f0 100644 --- a/apps/rlark/docs/user-guide/jobs.md +++ b/apps/rlark/docs/user-guide/jobs.md @@ -14,12 +14,12 @@ For each role: 2. Choose one GPU or embodied-device specification from the list, which shows available/total devices and nodes, then set the shared per-Worker resource request. Set the request to `0` for debugging workloads that should keep the placement constraint without requesting the selected device. 3. Choose one scheduling mode: - **Automatic selection**: enter the desired Worker count. The console validates the total request against schedulable capacity and selects eligible nodes. - -- **Select nodes**: click eligible nodes or drag across node cards. Each selected node creates one Worker; click or drag across selected nodes again to remove them. + - **Select nodes**: click eligible nodes or drag across node cards. Each selected node creates one Worker; click or drag across selected nodes again to remove them. Changing the per-Worker resource request afterwards keeps every selected node that still has enough free capacity and only drops the nodes that no longer fit. +4. Review the shared placement summary, then configure the image, prepare script, environment variables, and storage mounts. When cloning a Job, the scheduling mode is preserved. A role that did not pin specific nodes remains in automatic-selection mode instead of being converted -to manual node selection. 4. Review the shared placement summary, then configure the image, prepare script, environment variables, and storage mounts. +to manual node selection. Submit the Job, then open Job details to verify the running state and inspect Workers. @@ -27,6 +27,13 @@ After selecting a role, its resource summary shows the GPU or embodied-device mo When a Worker is Pending, hover or focus the information icon beside that Worker's status. RLark reads the related data-plane Pod, Task, or node events and converts recognized Kubernetes events into concise pending reasons in the current UI language. The mapping covers common scheduling, node pressure and availability, image-pull, volume, runtime environment, container-creation, and health-check issues. Unrecognized events use a generic pending label instead of exposing raw runtime messages. Image-pull progress remains separate and shows byte or percentage progress only when the runtime reports it. +The tooltip reports the current blocking reason rather than a raw event log: + +- `Pulling` and `Pulled` are per-container point-in-time events, and a Worker Pod also pulls platform-injected sidecar images. Only the newest pull-progress event stays in the current reasons; older ones move to the **Earlier Events** section. If the image pull failed, or a warning event is at least as recent as the newest progress event, every pull-progress event moves there too, so a successful sidecar pull can no longer read as "Image pulled" next to a failing Worker. +- kubelet reports both image-pull failures and container start failures as `Failed`. RLark narrows that reason using the event message, so an unusable image reads as "Invalid image name" or "Image pull failed" instead of "Worker startup failed". +- Repeated events are collapsed by reason, keeping the most recent occurrence, and the list is ordered newest first. The **Earlier Events** section keeps superseded entries visible, dimmed and struck through, for troubleshooting. + + ## Job Types RLark supports the following job types, each pre-configured with default roles tailored to the workload: @@ -56,10 +63,12 @@ For each worker role, configure the following: - **Init Scripts** — Commands that run before the main entrypoint. Useful for environment setup, dependency installation, or data preparation. -- **Storage Mounts** — Attach storage to worker containers. Two types are supported: +- **Storage Mounts** — Attach storage to worker containers. The Job editor can create two types: - **hostPath**: Mount a directory from the host node's filesystem. Its data is not deleted by Job lifecycle actions. - **PVC** (PersistentVolumeClaim): Mount Kubernetes storage using the selected storage class. The current Web UI writes `ephemeral.volumeClaimTemplate`, so these PVCs are Pod-scoped and are deleted and recreated with their Pods. Imported or older Tasks using `persistentVolumeClaim` and the deprecated PVC maps follow the retained Task-managed compatibility lifecycle instead. + Jobs submitted through the API may use other Kubernetes-native volume types, such as `emptyDir`, `configMap`, or `secret`. The Job editor cannot create or modify these volumes, but it lists them read-only in the role's mount list and on the Job detail page and preserves their original definitions when you edit or clone the Job. Removing the mount is the only change the editor supports for them. + > **Screenshot note:** Screenshots are from an example environment. Resource names and data are illustrative; your environment will differ. ![Worker configuration](../images/ui/create-job-worker-configuration.png) @@ -68,13 +77,13 @@ For each worker role, configure the following: Configure settings that apply to all workers in the job: -- **Header Role** — Select one role as the Header role. For Ray workloads, configure this role with exactly one Worker; it coordinates distributed training and its IP address is communicated to all other Workers. +- **Header Role** — Select one role as the Header role. For Ray workloads, configure this role with exactly one Worker; it coordinates distributed training and its IP address is communicated to all other Workers. Roles whose replica count is greater than 1 are disabled in this selector. If you later raise the replica count of the selected Header role, the console falls back to the first single-Worker role so the wizard stays correctable. - **Cross-Cluster Network Domain** — If network domains are configured, the console automatically enables the first domain by name for every Job, regardless of Worker placement. No domain is added when none is configured. - **SSH Public Keys** — Select the public key written to the Job's workload SSH configuration. Built-in Ray-on-Kubernetes workloads deliver it through a Task-owned Secret mounted at `/var/run/rlark/ssh/authorized_key` in each worker main container and start an optional `rlark-tools sshd` service. SSH failure does not fail the Task. Other workload implementations do not gain SSH merely by setting this field. -- **Run Command** — The main training script or entrypoint command. This is executed after any init scripts complete. +- **Run Command** — The main training script or entrypoint command. This is executed after any init scripts complete. After the run command finishes on the Header node, the Pod does not exit; it continues blocking to keep the Ray cluster running until the Job is manually stopped or deleted. - **TensorBoard** — Enable TensorBoard monitoring for the job. Configure the log directory path and the port on which TensorBoard will be served. @@ -163,7 +172,7 @@ nvidia-smi # Check GPU status (if GPU is allocated) ### File Transfer -The key button in the Worker list copies the SSH connection command. It requires an administrator-configured SSH jump host; when that setting is missing, the button is disabled and explains why instead of copying empty content. +The key button in the Worker list copies the SSH connection command. It requires an administrator-configured SSH jump host; when that setting is missing, the button is disabled and explains why instead of copying empty content. The copied command uniformly forwards through the node sshd on port 2222 to reach the target Pod. In non-HTTPS environments, the copy function automatically falls back to a compatible approach and is not affected by browser Clipboard API restrictions. The WebTerminal supports file upload and download: @@ -223,6 +232,10 @@ Deleting a Job cleans up its active workload resources, with these storage excep - **Edit** — Modify a stopped job's configuration (roles, resources, commands) and resubmit. - **Clone** — Create a copy of an existing job configuration to use as a template for a new job. +Both actions prefill the editor from the source Job and keep its roles, including custom roles that are not part of the selected job type's default roles. For each role the editor restores the container image, run command, prepare script, environment variables, replica count, CPU, memory and device requests, scheduling mode, and storage mounts; volume types the editor cannot edit keep their original definitions. Changing the job type after prefilling only updates the job type label and leaves the role list untouched: the prefilled roles are configuration you already confirmed, so add any default roles of the new type manually. + +The source cluster is not stored on the Job itself. The editor resolves it from each role's node selector, so verify the cluster selection before resubmitting. + ## Preflight Checklist Before submitting a job, verify the following: diff --git a/apps/rlark/docs/user-guide/ssh-keys.md b/apps/rlark/docs/user-guide/ssh-keys.md index 2061943..a6ce2f4 100644 --- a/apps/rlark/docs/user-guide/ssh-keys.md +++ b/apps/rlark/docs/user-guide/ssh-keys.md @@ -16,6 +16,8 @@ Use this guide to register a public key for RLark SSH bastion authentication and RLark validates the public-key format and requires both the SSH username and public key content to be globally unique. The form shows an inline error before upload when either value already exists. The page also shows the configured jump host command when the Server SSH address is available. +The connect command and every listed public key provide a copy button. On success the button briefly turns into a green check, matching the other copy buttons in the console; no success feedback is shown when the copy fails. When the console is served over HTTP (a non-secure context) the browser does not expose the async clipboard API, so copying automatically falls back to a compatible path and does not require switching to HTTPS. + ## Task 2: Select a Key for a Job 1. Open **Jobs** and start creating a Job. @@ -34,14 +36,18 @@ RLark validates the public-key format and requires both the SSH username and pub Each selected public key is written into the applicable Task template's `rlinf.io/ssh/v1` Environment. Registering or changing a record on the SSH Keys page alone does not modify existing Jobs or Tasks. Job details display multiple configured keys in a scrollable list. +Because a Job stores raw public keys rather than a reference to the key record, a key can outlive its owner record. Job details label such a key `User deleted` instead of a user name, with a tooltip explaining that the owner record was removed from SSH Key Management while the key stays injected in running Pods. Cloning, editing, or edit-and-restarting a Job drops those keys from the draft, because the selection list cannot render or deselect a key that has no owner record; the form reports how many keys were removed so you can add a replacement before submitting. + For built-in Ray-on-Kubernetes workloads, the Agent stores the Task's selected public key in a Task-incarnation-owned Secret, mounts its `authorized_key` item read-only at `/var/run/rlark/ssh/authorized_key` in each main container, and starts `rlark-tools sshd` as an optional supervised service. The mount does not use `subPath`, and `sshd` reopens the file for every authentication attempt. An empty file disables workload key authentication. An SSH service failure can make the Pod unreachable over SSH, but does not stop the Task. This automatic service lifecycle does not apply to Docker, Raw, or arbitrary custom workload implementations. +The Job Controller sets the SSH port to 2222 for all Tasks. The frontend-generated SSH command forwards through the node sshd on port 2222 to reach the target Pod. + ## Task 3: Connect Through the Bastion After an administrator configures the Server SSH endpoint, use the command displayed by the console or copy the Worker-specific command from Job or Node details. A typical Worker command is: ```bash -ssh -J : root@ +ssh -J : root@ -p 2222 ``` The outer SSH username must match the username attached to the registered key. The target container must provide its SSH service and accept the selected key. diff --git a/apps/rlark/docs/user-guide/storage.md b/apps/rlark/docs/user-guide/storage.md index d149513..22aa02c 100644 --- a/apps/rlark/docs/user-guide/storage.md +++ b/apps/rlark/docs/user-guide/storage.md @@ -2,13 +2,14 @@ ## Storage Types -The Job editor exposes two storage choices: a host directory or a PVC-backed storage mount. Kubernetes-native PodTemplate volumes define their lifecycle: +The Job editor can create two storage choices: a host directory or a PVC-backed storage mount. A Job submitted through the API may also use other Kubernetes-native PodTemplate volumes, which the editor shows read-only. Kubernetes-native PodTemplate volumes define their lifecycle: | Type | Use Case | Lifecycle | |------|----------|-----------| | Host Directory | Data already on the node, high I/O | Job lifecycle actions do not delete data | | Object Storage (Pod-scoped ephemeral PVC) | The form written by the current Web UI for a PVC-backed mount | Kubernetes creates and removes the PVC with its Pod | | Existing persistent PVC reference | A Task volume uses `persistentVolumeClaim.claimName` | RLark neither creates nor deletes the referenced PVC; its administrator controls the lifecycle | +| Other Kubernetes-native volume (`emptyDir`, `configMap`, `secret`, and so on) | Submitted through the API; the Job editor only displays it | The volume type defines it; RLark writes back the original definition when a Job is edited or cloned | ## Host Directory @@ -56,6 +57,7 @@ Verify the storage chain: - Existing `persistentVolumeClaim`: RLark does not create, adopt, mutate, or delete the referenced PVC. Its lifecycle and data retention are external to the Task. - Pod-scoped `ephemeral.volumeClaimTemplate`: deleting a Worker Pod deletes its ephemeral PVC; a new Pod receives a new PVC. - HostPath: lifecycle actions do not delete node data. +- Other Kubernetes-native volumes: the Job editor cannot create or modify them. Editing or cloning a Job writes the original volume definition back unchanged, so the volume type keeps its own lifecycle. An existing PVC reference must include `claimName`, and the claim must already exist in the workload namespace. Kubernetes reports mount or binding failures through the workload and Pod status. Deleting a referenced PVC is an external administrative action; confirm that no workload uses it and account for the bound PV's reclaim policy. diff --git a/apps/rlark/docs/zh/admin-guide/agent.md b/apps/rlark/docs/zh/admin-guide/agent.md index ccec77a..df184ca 100644 --- a/apps/rlark/docs/zh/admin-guide/agent.md +++ b/apps/rlark/docs/zh/admin-guide/agent.md @@ -130,17 +130,19 @@ kubectl get cluster -o jsonpath='{.spec.agentNamespace}{"\n"}' 完整参数列表参见[配置项参考](../reference/configuration.md#rlark-agent)。 -Runtime/Workload Coordinator 现在是 Deployment、DaemonSet 和 StatefulSet Task 的生产写入者，负责执行有序 Runtime 操作。Runtime push controller 只入队调谐 hint；通过验证的定向观测和 Task Status 更新均由 pull controller 执行。 +Runtime/Workload Coordinator 现在是 Deployment 和 StatefulSet Task 的生产写入者，负责执行有序 Runtime 操作。Runtime push controller 只入队调谐 hint；通过验证的定向观测和 Task Status 更新均由 pull controller 执行。 + +Ray-on-Kubernetes Workload Adapter 支持 Deployment 和 StatefulSet carrier。 Cluster Agent 启动后会立即执行一次修复扫描，随后按配置的抖动周期重复。每轮分页扫描分配给同一 Host 或符合该 Host 条件的 management Task，预算上限为 10,000 个 Task，并产生完整调谐 hint；扫描本身不直接修改 Task 或 Runtime 对象。Management API 的临时 List 失败会记录日志并在后续轮次重试，不会停止 Agent；trigger bridge 背压会按轮汇总，但不会阻止扫描继续访问后续 Task 和分页。禁用扫描会移除 watch event 丢失后的这一恢复路径，只有在其他运维机制能够保证最终收敛时才应这样做。 -升级 Agent 时，Coordinator 可以识别旧 Task controller 创建、但尚未携带 Task UID Annotation 的主工作负载。它只会接管名称相同且现有元数据与 Task 不冲突的 Deployment、DaemonSet 或 StatefulSet；该兼容规则不会扩展到任意辅助资源。首次接管以及 workload 仍处于该 legacy adopted generation 时的后续调谐只更新 identity 和 revision metadata，保留运行中的 Pod 模板，避免升级本身触发 rollout。后续 Task generation 将 workload 建立为 canonical-owned 后，即使当前 Task generation 未变化，完整调谐也会修复 Agent 管理的 replicas 和 Pod-template 字段的带外漂移。 +升级 Agent 时，Coordinator 可以识别旧 Task controller 创建、但尚未携带 Task UID Annotation 的主工作负载。它只会接管名称相同且现有元数据与 Task 不冲突的 Deployment 或 StatefulSet；该兼容规则不会扩展到任意辅助资源。首次接管以及 workload 仍处于该 legacy adopted generation 时的后续调谐只更新 identity 和 revision metadata，保留运行中的 Pod 模板，避免升级本身触发 rollout。后续 Task generation 将 workload 建立为 canonical-owned 后，即使当前 Task generation 未变化，完整调谐也会修复 Agent 管理的 replicas 和 Pod-template 字段的带外漂移。 不要通过修改 `spec.runtime.kind` 或 `spec.runtime.name` 迁移已经 claim 的 Task。Agent 会冻结已 claim Runtime：这些字段变化时保留旧 workload，不应用该 generation 中的其他 Spec 变化，并等待恢复原值，或者等待 Task 删除后重建。已 claim 且未删除的 Task 出现解码、校验或 implementation 注册失败时，也会保留 claim 和现有 workload，而不会执行 cleanup。删除仍按已 claim Runtime 和 Task identity 执行 cleanup。升级期间，对于带 cleanup finalizer 但没有 claimant annotation 的 legacy Task，只有 Task 正在删除或者当前 Agent 能够执行时才会接管；未删除且 selector 指向其他 Agent 的 Task 会保持不动。完整 ownership 契约参见 [Runtime、Workload 与 Environment 边界](../developer-guide/runtime-provider.md)。 -上层业务不应把修改已claim Kubernetes Task的`spec.workload.kind`作为常规工作流；更换carrier kind时应以新UID替换Task，或使用Job restart/replacement流程。如果直接修改仍然到达Agent，才会触发Deployment、DaemonSet和StatefulSet之间的窄化、可恢复兼容路径：Agent先清理并确认旧carrier及旧selector下所有`DeleteOnTaskCleanup`辅助对象均已消失，其中包括Ray head Service，同时保留共享Ray ConfigMap；引用的外部PVC不属于RLark ownership。随后切换claim中冻结的Workload selector，并在后续调谐创建新carrier。ownership冲突或证据不完整会阻止迁移。该fallback可能中断workload，既不是跨Runtime迁移，也不提供事务回滚；它用于避免资源泄漏和claim metadata失真，不能替代Task replacement。 +上层业务不应把修改已claim Kubernetes Task的`spec.workload.kind`作为常规工作流；更换carrier kind时应以新UID替换Task，或使用Job restart/replacement流程。如果直接修改仍然到达Agent，才会触发Deployment和StatefulSet之间的窄化、可恢复兼容路径：Agent先清理并确认旧carrier及旧selector下所有`DeleteOnTaskCleanup`辅助对象均已消失，其中包括Ray head Service，同时保留共享Ray ConfigMap；引用的外部PVC不属于RLark ownership。随后切换claim中冻结的Workload selector，并在后续调谐创建新carrier。ownership冲突或证据不完整会阻止迁移。该fallback可能中断workload，既不是跨Runtime迁移，也不提供事务回滚；它用于避免资源泄漏和claim metadata失真，不能替代Task replacement。 -Task 删除、停止和重启会保持等待，直到确认所有该 Task 独占且标记为 `DeleteOnTaskCleanup` 的对象均已消失。这包括主 Deployment、DaemonSet 或 StatefulSet，以及 Ray head Task 由UID标识的head Service。共享Ray启动ConfigMap会被保留；引用的外部PVC不进入cleanup inventory。快照或删除失败时会保留Task finalizer，以便后续安全恢复。 +Task 删除、停止和重启会保持等待，直到确认所有该 Task 独占且标记为 `DeleteOnTaskCleanup` 的对象均已消失。这包括主 Deployment 或 StatefulSet，以及 Ray head Task 由UID标识的head Service。共享Ray启动ConfigMap会被保留；引用的外部PVC不进入cleanup inventory。快照或删除失败时会保留Task finalizer，以便后续安全恢复。 当RLark拥有的selector、port、Service type、`publishNotReadyAddresses`或标准metadata发生漂移时，Ray head Service使用Replace语义。Agent会按UID校验并删除当前Service，确认对象消失后在后续reconcile中重建；这可能短暂中断访问，也可能分配不同的Service地址或端口。Kubernetes物化的默认值和分配字段，包括ClusterIP、IP family字段、NodePort、默认protocol和target port，以及无关metadata，不属于漂移投影，不会单独触发Replace。 diff --git a/apps/rlark/docs/zh/admin-guide/cluster-delisted-migration.md b/apps/rlark/docs/zh/admin-guide/cluster-delisted-migration.md new file mode 100644 index 0000000..3721209 --- /dev/null +++ b/apps/rlark/docs/zh/admin-guide/cluster-delisted-migration.md @@ -0,0 +1,26 @@ +# 迁移 Cluster 下架状态 + +RLark 现在使用 `Cluster.spec.delisted` 表示管理员已下架集群。早期版本使用 `Cluster.spec.deleted` 表示相同的持续意图。 + +## 升级顺序 + +1. 先部署同时接受 `spec.deleted` 与 `spec.delisted` 的过渡版 CRD。 +2. 在从 CRD 移除 `spec.deleted` 前，针对承载 kcp 的 Kubernetes 集群执行迁移脚本： + +```bash +KUBECONFIG=~/.kube/host.kubeconfig \ + ./hack/migrate-cluster-deleted-to-delisted.sh rlark-system kcp-0 +``` + +3. 使用 dry-run 确认不存在遗留资源： + +```bash +KUBECONFIG=~/.kube/host.kubeconfig \ + ./hack/migrate-cluster-deleted-to-delisted.sh rlark-system kcp-0 --dry-run +``` + +4. 部署移除 `spec.deleted` 并且只使用 `spec.delisted` 的版本。 + +脚本会从 `rlark-kcp-kubeconfig` 读取 kcp 管理员 kubeconfig，并对所选 kcp Pod 建立端口转发，因此实际操作的是 kcp API，而不是宿主 Kubernetes API。脚本仅处理 `spec.deleted=true` 的 Cluster 资源，将 `spec.delisted` 设置为 `true`，并通过 merge patch 删除旧字段。脚本可重复执行；成功迁移后再次运行不会产生变更。 + +迁移完成前，请勿部署会拒绝或裁剪 `spec.deleted` 的 CRD，否则旧字段可能会在其语义转移到 `spec.delisted` 前被移除。 diff --git a/apps/rlark/docs/zh/api/reference.md b/apps/rlark/docs/zh/api/reference.md index 5529d6a..7b445ba 100644 --- a/apps/rlark/docs/zh/api/reference.md +++ b/apps/rlark/docs/zh/api/reference.md @@ -94,11 +94,11 @@ Gateway 虽注册了 `POST /api/v1/certificates/revoke`，但该接口尚未实 | `GET, POST` | `/api/v1/storage/storageclass` | | `PUT, DELETE` | `/api/v1/storage/storageclass/{name}` | | `GET` | `/api/v1/storage/storageclass/provider` | -| `GET` | `/api/v1/storage/storageclass/{name}/{cluster}/list` | -| `POST` | `/api/v1/storage/storageclass/{name}/{cluster}/upload` | -| `GET, DELETE` | `/api/v1/storage/storageclass/{name}/{cluster}/object/{key}` | +| `GET` | `/api/v1/storage/storageclass/{name}/list` | +| `POST` | `/api/v1/storage/storageclass/{name}/upload` | +| `GET, DELETE` | `/api/v1/storage/storageclass/{name}/object/{key}` | -StorageClass 接口管理控制面声明。Replication 和 Delivery 控制器会将生成的 rclone CSI 资源异步分发到已配置的目标集群；列表返回的是已管理声明，而非实时查询 Agent 的结果。 +StorageClass 接口管理控制面声明。Replication 和 Delivery 控制器会将生成的 rclone CSI 资源异步分发到已配置的目标集群；列表返回的是已管理声明，而非实时查询 Agent 的结果。文件路由会直接访问声明的存储桶。 ## Addon diff --git a/apps/rlark/docs/zh/architecture.md b/apps/rlark/docs/zh/architecture.md index 6810b30..05adf5a 100644 --- a/apps/rlark/docs/zh/architecture.md +++ b/apps/rlark/docs/zh/architecture.md @@ -122,7 +122,7 @@ graph LR **Pull 控制器**（以 Task 为例）： 1. 监听管理面 Task CR 的创建/更新/删除（带 finalizer 保护） -2. 将 Task 解码为 execution snapshot，并选择 Kubernetes Runtime 和面向 Ray 的 Kubernetes Workload Adapter；旧版 Deployment、DaemonSet 或 StatefulSet discriminator 只选择其 carrier kind +2. 将 Task 解码为 execution snapshot，并选择 Kubernetes Runtime 和面向 Ray 的 Kubernetes Workload Adapter；carrier 声明选择 Deployment 或 StatefulSet 3. 编译声明式对象计划，由 Adapter 选择 ownership、lifecycle、owned field、update mode 和 typed Runtime reconcile profile；向 Adapter 请求有界 identity inventory，读取对应 Runtime 快照，再计算所需操作 4. 按顺序执行幂等 Apply/Delete；`Replace` 对象先按 UID 校验并删除，再由后续 reconcile 重建；运行中的 Task 在期望 generation 变化后也会重新调谐 5. 在编译期间应用 Ray bootstrap 等 Workload 自有 Pod 模板修改，再解析显式 Environment requirement，并且只向 Adapter 授权的 PodTemplate target 绑定跨 Domain Network Sidecar diff --git a/apps/rlark/docs/zh/concepts.md b/apps/rlark/docs/zh/concepts.md index b5eaedd..87b0b2e 100644 --- a/apps/rlark/docs/zh/concepts.md +++ b/apps/rlark/docs/zh/concepts.md @@ -176,7 +176,7 @@ Job 是用户**直接操作的核心资源**，代表一个完整的 RL 训练 分发任何 Task 前，Job Controller 会把所有模板作为一个整体逐一校验。模板名称必须非空且唯一，`runtime.kind`、workload kind/version 与 carrier type/version 必须是非空的开放标识符。对于当前 Job 到 Ray-on-Kubernetes 路径，controller 还要求提供 Pod 模板，并且其中必须且只能有一个名为 `main` 的普通容器。这是实现能力校验，不是通用 Task CRD 约束：CRD 允许省略 `workload.template`，其他 Workload Adapter/Runtime 组合自行定义要求。任一模板无效时，controller 不会创建其中任何 Task，而是在 Job 上报告 `Validated=False` 并通过消息指出失败字段路径；它不会静默跳过单个模板。Agent 会在 Runtime effect 前独立校验所选实现。Job Controller 不校验 Ray head 数量或 head 副本拓扑。 -Web UI 保持现有的 Job 和 Workflow 创建、编辑、克隆、生命周期、存储及 Worker 交互不变，但会直接按生成的 CRD 合同序列化数据。新请求使用 `runtime`、`workload`、`carrier`、`placement` 和 `environments`；生命周期操作写入 `spec.suspended`；存储选择写成 Kubernetes 原生 Pod 模板 volume。UI 不再发送已移除的 `agentType`、嵌套 `kubernetes.workload`、`stopped`、`pvcStorageMap` 或 `pvcSizeGbMap` 结构。 +Web UI 保持现有的 Job 和 Workflow 创建、编辑、克隆、生命周期、存储及 Worker 交互不变，但会直接按生成的 CRD 合同序列化数据。新请求使用 `runtime`、`workload`、`carrier`、`placement` 和 `environments`；生命周期操作写入 `spec.suspended`；存储选择写成 Kubernetes 原生 Pod 模板 volume。表单无法表达的卷类型（例如 `emptyDir`、`configMap`、`secret`）以只读方式展示，并在编辑或复制任务时按源定义原样保留。UI 不再发送已移除的 `agentType`、嵌套 `kubernetes.workload`、`stopped`、`pvcStorageMap` 或 `pvcSizeGbMap` 结构。 ### 关键属性 @@ -364,7 +364,7 @@ status: | 形态 | 说明 | 适用场景 | |------|------|----------| -| `Kubernetes` | 使用 Deployment、DaemonSet 或 StatefulSet 作为 Kubernetes carrier 运行 RLark/Ray Task | GPU 集群大规模训练 | +| `Kubernetes` | 使用 Deployment 或 StatefulSet 作为 Kubernetes carrier 运行 RLark/Ray Task | GPU 集群大规模训练 | | `Docker` | 计划中的 Provider，把统一 Pod template lowering 为 Docker 容器；当前未实现、未广告 | 端侧设备（机械臂、传感器、摄像头） | | `Raw` | 计划中的 Provider，把统一 Pod template lowering 为宿主机进程；当前未实现、未广告 | 裸金属服务器、嵌入式设备 | @@ -407,7 +407,7 @@ Pod CR 是数据面 Pod 的**控制面镜像**，由 Agent 的 Push 控制器上 ```mermaid graph TD job -->|"1:N"| task["Task
(Namespaced: agent-{id})
任务执行单元"] - task -->|"1:1 (K8s workload)"| workload["Deployment /
DaemonSet /
StatefulSet
(本地 k8s 集群)"] + task -->|"1:1 (K8s workload)"| workload["Deployment /
StatefulSet
(本地 k8s 集群)"] workload -->|"1:N"| pod["Pod + Sidecar
Agent Push 上报 → Pod CR"] domain["Domain
(Cluster scoped)
网络隔离"] -->|"1:N (每个集群一个)"| dp["DomainPeer
(Namespaced: agent-{id})
Pod 路由表"] node["Node
(Namespaced)
计算节点信息"] diff --git a/apps/rlark/docs/zh/developer-guide/api-conventions.md b/apps/rlark/docs/zh/developer-guide/api-conventions.md index c94dca5..26f3df9 100644 --- a/apps/rlark/docs/zh/developer-guide/api-conventions.md +++ b/apps/rlark/docs/zh/developer-guide/api-conventions.md @@ -245,7 +245,7 @@ Kubernetes 错误是映射而非包裹：`NotFound` → 404，`AlreadyExists` | `order` | `field [asc\|desc]`；field 须匹配 `^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*$`，方向无法识别时直接拒绝，而不是静默按升序排序。 | | `fieldSelector` | 逗号分隔的 `key=value` 或 `key!=value`；每个 key 须匹配与 `order` 相同的点分标识符模式。 | | `labelSelector` | 数据库链路为逗号分隔的 `key=value` 或 `key!=value`，Kubernetes 链路为 Kubernetes label selector。key 须是合法的 qualified name。 | -| `tagSelector` | 逗号分隔的 `key=value`。在服务端窗口之后应用。 | +| `tagSelector` | 逗号分隔的 `key=value`。同一 key 下的多个值取「或」，不同 key 之间取「且」。在服务端窗口之后应用。 | | `continue` | 不透明的 Kubernetes continue token。仅 Kubernetes 链路。 | 选择器路径与标签 key 来自查询串，因此在数据库链路上一律作为语句参数绑定，绝不拼入 SQL 文本。上表的模式校验在入口处执行，属于纵深防御：无意义的路径在抵达数据库之前就以 400 拒绝。 diff --git a/apps/rlark/docs/zh/developer-guide/runtime-provider.md b/apps/rlark/docs/zh/developer-guide/runtime-provider.md index 9b73bd6..0add048 100644 --- a/apps/rlark/docs/zh/developer-guide/runtime-provider.md +++ b/apps/rlark/docs/zh/developer-guide/runtime-provider.md @@ -60,7 +60,7 @@ Workload 与 Runtime 概念保持正交，但不能任意组合。只有显式 Agent 会在启动装配阶段拒绝该注册。Kubernetes 当前只注册一个 `(rlinf.io/ray, v1, Kubernetes)` implementation。Ray Adapter 独立解码 `workload.carrier`，支持版本为 `v1` 的 `rlinf.io/kubernetes-deployment`、 -`rlinf.io/kubernetes-daemonset` 和 `rlinf.io/kubernetes-statefulset` carrier。 +`rlinf.io/kubernetes-statefulset` carrier。 它们不是通用 Kubernetes 原生 workload 实现。 未 claim Task 会先完成解码和精确 Registry 检查，之后才能写入任何 claim metadata。 @@ -72,7 +72,7 @@ implementation，并在不依赖当前 Workload Spec 的情况下按 Task identi 因此在 API 降级或只完成部分迁移的更新后仍可恢复这些清理流程。完整的 per-Runtime Bundle/Factory 仍属于后续装配工作，本文不会把它描述为已经存在。 -Deployment、DaemonSet 和 StatefulSet 是单个 Ray Adapter 注册内的 carrier 选择， +Deployment 和 StatefulSet 是单个 Ray Adapter 注册内的 carrier 选择， 不是 Workload 选择入口。 该 compatibility tree 有意归 Task controller 所有，不属于 Runtime-neutral @@ -143,7 +143,7 @@ cleanup，因为此时已无法证明 Task UID 和 claimed identity。 - Workload 自有模板修改和 workload identity，在 Workload 编译期间、渲染期望 Kubernetes 对象之前仅执行一次；跨 Domain Network 注入改由 Environment binding 完成； - 镜像拉取 Secret、PVC 与 Ray 资源； - 声明式 workload SSH requirement；Secret 交付和 Pod template mount 由 Environment 层负责； -- 渲染选中的 Deployment、DaemonSet 或 StatefulSet workload，再由 Coordinator 和 Runtime Provider 应用。 +- 渲染选中的 Deployment 或 StatefulSet workload，再由 Coordinator 和 Runtime Provider 应用。 旧 `Operations` Adapter 仅作为有测试覆盖的兼容端口保留；生产装配使用 Ray-on-Kubernetes Adapter，不会调用其中的 `ApplyWorkload` 方法。 @@ -159,7 +159,7 @@ Process declaration 是 portable Plan 合同，会被深拷贝并计入 Adapter 在可选的用户 prepare-script 边界之后、执行任何 Ray stop/start 命令之前，`ray-launcher` 会确认配置的 Ray CLI 可解析且可执行。CLI 缺失或不可执行会使 primary 失败。Head 上通过 embed 交付的 Ray SDK 集群检查为保持兼容仍是非致命检查，但随后的 `ray status` 失败属于致命错误。Worker 上无论 `ray start --block` 启动失败，还是启动后 Ray 以非零状态退出，都会使 primary 失败；普通进程退出时，Ray 退出码会经 launcher 和 supervisor 保留下来。 -Workload 语义由 `spec.workload.kind` 和 `spec.workload.version` 选择，任意 JSON `spec.workload.parameters` 必须由选中的 Adapter 严格解码。`spec.workload.carrier` 独立包含 Runtime-specific carrier 的 `type`、`version` 和任意 JSON `parameters`。Deployment 与 StatefulSet carrier 只接受 `{replicas}`；DaemonSet carrier 只接受空对象。Environment 使用相同的 type/version/parameters envelope 组成有序列表，并且允许重复；是否要求唯一由选中的 Environment resolver 按自身合同决定。未知类型、版本、非法参数或 Runtime 不支持的能力都必须在 Provider effect 前失败。 +Workload 语义由 `spec.workload.kind` 和 `spec.workload.version` 选择，任意 JSON `spec.workload.parameters` 必须由选中的 Adapter 严格解码。`spec.workload.carrier` 独立包含 Runtime-specific carrier 的 `type`、`version` 和任意 JSON `parameters`。Deployment 与 StatefulSet carrier 只接受 `{replicas}`。Environment 使用相同的 type/version/parameters envelope 组成有序列表，并且允许重复；是否要求唯一由选中的 Environment resolver 按自身合同决定。未知类型、版本、非法参数或 Runtime 不支持的能力都必须在 Provider effect 前失败。 Task decoder 会通用映射每个 Environment envelope，不枚举内置类型：它只检查必需的公共字段，保留顺序和重复声明，深拷贝参数，并把特定类型的唯一性、version 与参数校验交给选中的 Runtime resolver。Kubernetes Environment resolver 通过不可变的 type-to-resolver registry 安装；空 type、nil resolver 或重复 type 注册会在构造时失败。组合式 resolver 按类型首次出现顺序分组，并把同一类型的全部声明按原始顺序一次性交给对应 resolver。Network、SSH 与 TensorBoard 当前会自行拒绝多个声明；未来的 Environment 合同可以注册接受并解析多个声明的 resolver。因此未知类型可以完成 decode，但会在 Provider observation 或 effect 前的 resolve 阶段 fail closed。Portable execution plan 保留同样的 Environment envelope，不再把 requirement 拆分成 network、secret、tool、service、storage、device 或 config 字段。每个 plan requirement 包含 `name`、开放字符串 `type` 与 `version`、复制后的 `parameters` 以及 consumer selectors。SSH authorized public keys 只保留在本地 execution declaration 中，不进入 portable requirement parameters 或 workload transport；SSH Handler 通过 material boundary 获取它们。 @@ -251,13 +251,13 @@ Runtime 的 `ListCandidateObjects`、旧版 `ListManaged` 和 `ListInstances` 选定的 Workload Adapter 根据冻结的 Task 和 implementation identity 生成有界 inventory。候选项按创建顺序声明。identity cleanup 以 `LifecycleDeleteOnTaskCleanup` 作为删除授权，每轮删除当前仍存在的最后一个精确名称 cleanup candidate，即按逆序清理；它不维护 Kubernetes kind allow-list。Ray-on-Kubernetes Adapter 先声明 SSH Secret，再声明其 workload 消费方，使 workload 对象先于 Secret 消失。它不声明 PVC candidate：引用的外部 PVC 不属于 RLark ownership，generic ephemeral PVC 则随 Pod 生命周期管理。Kubernetes Provider 只执行 Adapter 声明，不计算 Ray 名称，也不推断 Ray role。候选名称和标签只能定位对象，不能证明 ownership；Provider 返回对象前仍会校验标准 Task identity，并在适用处校验对象 UID。任一查询失败都会使完整快照原子失败。实例观测会对 workload root 执行 UID 校验，只列出该 workload 选中的 Pod，并使用 `involvedObject.uid` field selector 查询 Pod Event。 -每个 desired Kubernetes object 都会显式选择 typed reconcile profile，而不是让 Provider 根据 manifest kind 推断更新策略。当前 Ray Adapter 为 Deployment、DaemonSet 和 StatefulSet carrier 选择 workload projection，为 head Service 选择 Service projection，为 bootstrap ConfigMap 选择 shared-field projection。 +每个 desired Kubernetes object 都会显式选择 typed reconcile profile，而不是让 Provider 根据 manifest kind 推断更新策略。当前 Ray Adapter 为 Deployment 和 StatefulSet carrier 选择 workload projection，为 head Service 选择 Service projection，为 bootstrap ConfigMap 选择 shared-field projection。 Runtime-owned 不可变 registry 会分派显式选择的版本化 executor，并验证 typed manifest 是否与该 profile 匹配；profile 缺失、未知、重复或不匹配都会被拒绝。每个 executor 为一个有界 Kubernetes 机制同时实现 validation、comparison 和 update preparation，例如比较 Service owned projection。选择哪种机制表达 Workload 契约仍由 Adapter 负责。 在安排 Service Replace 前，Coordinator 通过该 registry 只比较 Service profile 拥有的字段。API Server 物化和分配的 ClusterIP、ClusterIPs、IP family、NodePort、默认 protocol 和 target port、流量策略、session affinity，以及无关 metadata 都不会产生 Replace 漂移。内置 registry 仍刻意保持窄化，其 workload executor 不是通用 CRD update policy。 -Kubernetes Provider 在比较期望 Deployment、DaemonSet 或 StatefulSet 的 Pod template 与 API Server 返回的对象之前，会在期望 template 未指定相应字段时保守地保留常见的 Pod 和 container 具现化默认值。当前兼容集合覆盖 Pod restart policy、DNS policy、scheduler name、termination grace period、空 Pod security context，以及 container image-pull 和 termination-message 默认值。这样既不会覆盖显式期望值，也能避免常见的重复更新，但并未复现随 Kubernetes 版本变化的全部嵌套 container、probe、volume 或 ephemeral-container 默认规则。当前 Provider 将完整 Pod template 视为 Agent 管理字段；mutating admission webhook 或其他 controller 注入该 template 的字段不属于当前支持的 ownership 边界，可能被覆盖后再次注入。在实现明确的 Agent-owned 字段投影或 server-side apply ownership 契约前，集群不应对 RLark 管理的 workload template 应用此类变更。 +Kubernetes Provider 在比较期望 Deployment 或 StatefulSet 的 Pod template 与 API Server 返回的对象之前，会在期望 template 未指定相应字段时保守地保留常见的 Pod 和 container 具现化默认值。当前兼容集合覆盖 Pod restart policy、DNS policy、scheduler name、termination grace period、空 Pod security context，以及 container image-pull 和 termination-message 默认值。这样既不会覆盖显式期望值，也能避免常见的重复更新，但并未复现随 Kubernetes 版本变化的全部嵌套 container、probe、volume 或 ephemeral-container 默认规则。当前 Provider 将完整 Pod template 视为 Agent 管理字段；mutating admission webhook 或其他 controller 注入该 template 的字段不属于当前支持的 ownership 边界，可能被覆盖后再次注入。在实现明确的 Agent-owned 字段投影或 server-side apply ownership 契约前，集群不应对 RLark 管理的 workload template 应用此类变更。 Ray bootstrap ConfigMap 使用 RLark 保留固定名称，并跨 Task cleanup 保留。Adapter 在每个 desired object 中显式投影其 `data`、`binaryData` 和 Ray 脚本身份 annotation 的 ownership；Provider 只更新该投影，不识别 Ray 名称。脚本内容一致时保持零写，发生漂移时原地修复；其他 label 和 annotation 会保留。没有身份 annotation 的旧固定名称 ConfigMap 可以规范化，但 Ray role metadata 冲突时会保持不动并返回 Conflict。shared object 缺少 projection 或 projection 不受支持时会被拒绝，不会退化为整对象更新。运维人员不得把这些保留名称用于无关 ConfigMap，也不得让其他 manager 写入脚本字段。 @@ -305,7 +305,7 @@ Kubernetes 执行路径在有界的进程内 accumulator 中合并 desired-state 共享的 cluster-Agent resync 启动后立即执行首轮，通过分页扫描分配给同一 Host 或符合该 Host 条件的 management Task，每轮预算上限为 10,000 个 Task。扫描器与调谐共享 Host claim 和 eligibility 规则，而不是重复实现：有效 claim 必须包含 Agent finalizer；已 claim Task 即使 selector 漂移或进入删除流程，仍属于其 owning Host。后续扫描使用 `--task-runtime-resync-period` 的 80%–120% 抖动间隔；默认值为 `15m`，`0` 表示禁用，负值会在启动时被拒绝。Management API 的临时 List 失败会记录日志并在下一周期重试，不会终止 Agent；trigger bridge 背压按轮汇总，也不会阻止后续 Task 和分页被访问。扫描只产生 periodic 完整调谐 hint，不把进程内扫描进度当作期望状态。 -对于已经证明 ownership 的主 Deployment、DaemonSet 或 StatefulSet，正常 Apply 会在 root 顶层 annotation 写入 canonical Runtime 身份（ID/kind）、Workload 身份（type/version）、compiled/executable digest 和不含秘密的 binding revision。现有 Task name、namespace、UID、generation 与 domain annotation 仍是 ownership 和 lifecycle 兼容契约。Job restart annotation 不会复制到 Runtime 对象，也不会由 Agent 解释。metadata 只写在 root 对象顶层，不写入 Pod template，因此仅修复 metadata 不会触发 Pod rollout。缺失、格式错误或不匹配的 canonical revision metadata 不能授权定向观测，必须选择完整调谐。对于 canonical-owned workload，完整 Apply 只有在证明 Task ownership（包括 Task UID annotation）、UID-fenced reference 所指向的当前对象实例以及 selector 兼容后，才会修复整组 canonical execution revision。窄化兼容的 legacy workload 在不存在冲突 ownership metadata 时，可依据 name、namespace、kind 与 selector shape 进行接管；该接管会写入 canonical Task 身份和 revision metadata，但保留 Pod template。后续 canonical-owned Apply 即使在相同 Task generation 也可修复 revision，从而允许 identifier 规范化或 digest 算法升级自愈。Execution revision annotation 只是顺序与优化证据，本身不构成 ownership 证明。 +对于已经证明 ownership 的主 Deployment 或 StatefulSet，正常 Apply 会在 root 顶层 annotation 写入 canonical Runtime 身份（ID/kind）、Workload 身份（type/version）、compiled/executable digest 和不含秘密的 binding revision。现有 Task name、namespace、UID、generation 与 domain annotation 仍是 ownership 和 lifecycle 兼容契约。Job restart annotation 不会复制到 Runtime 对象，也不会由 Agent 解释。metadata 只写在 root 对象顶层，不写入 Pod template，因此仅修复 metadata 不会触发 Pod rollout。缺失、格式错误或不匹配的 canonical revision metadata 不能授权定向观测，必须选择完整调谐。对于 canonical-owned workload，完整 Apply 只有在证明 Task ownership（包括 Task UID annotation）、UID-fenced reference 所指向的当前对象实例以及 selector 兼容后，才会修复整组 canonical execution revision。窄化兼容的 legacy workload 在不存在冲突 ownership metadata 时，可依据 name、namespace、kind 与 selector shape 进行接管；该接管会写入 canonical Task 身份和 revision metadata，但保留 Pod template。后续 canonical-owned Apply 即使在相同 Task generation 也可修复 revision，从而允许 identifier 规范化或 digest 算法升级自愈。Execution revision annotation 只是顺序与优化证据，本身不构成 ownership 证明。 Binding revision 是标准化的 `sha256:` digest，输入包括 binding kind/name、可信 profile identity 与 revision、排序后的具体 target consumer、ownership、lifecycle、source revision、handler algorithm 和 value type；不包含 raw binding value 或 Secret 内容。计算 executable hash 时也会先把 Kubernetes Secret 的 `data` 和 `stringData` 替换为固定标记；Runtime comparator 仍获得真实 desired Secret，因此 key 轮换仍会触发原地 Secret 更新。`ResolvedBinding.Value` 只能承载受控、可序列化且非敏感的 option；raw key、token 和 credential 必须使用单独受限的进程内 material 机制，绝不能进入 portable Plan、transport、Status、诊断、error、hash 或 revision。生产诊断脱敏不能通过运行时配置关闭；observer 失败或 panic 不会改变调谐结果、Status 或 Runtime effect。接入新的诊断 payload shape 前，必须针对每个承载值的字段扩展并测试脱敏：当前通用 sanitizer 覆盖 environment `value`、名称含敏感词的 key，以及 Secret `data`/`stringData`，但不会推断 container `command`/`args` 中嵌入的秘密，也不会处理非敏感 key 下 `TypedOptions` value 中的秘密。基于 Environment 的 Network、workload SSH 和 TensorBoard mutation，以及 Runtime 强制提供的 RLark tools 与 supervisor assembly，均已进入生产装配。 @@ -320,7 +320,6 @@ Binding revision 是标准化的 `sha256:` digest，输入包括 binding kind/na | Task claim 与 finalizer | 支持 | 未实现 | 未实现 | | Deployment | 支持 | 不适用 | 不适用 | | StatefulSet | 支持 | 不适用 | 不适用 | -| DaemonSet | 支持 | 不适用 | 不适用 | | 容器/进程执行 | Kubernetes Pod | 仅有 Spec 骨架 | 仅有 Spec 骨架 | | 状态观察 | 支持 | 未实现 | 未实现 | | 运行时日志 | Kubernetes Pod fallback | 未实现 | 未实现 | diff --git a/apps/rlark/docs/zh/reference/configuration.md b/apps/rlark/docs/zh/reference/configuration.md index f71651d..41bb577 100644 --- a/apps/rlark/docs/zh/reference/configuration.md +++ b/apps/rlark/docs/zh/reference/configuration.md @@ -163,7 +163,6 @@ rlark-controller-manager \ | `--task-runtime-resync-period` | duration | `15m` | cluster 模式共享 Task Runtime 修复扫描的基础周期；实际间隔使用 80%–120% 抖动，`0` 表示禁用，负值无效；node-only 模式接受这个共享 flag，但不会运行扫描器 | | `--addon-pull-controller-workers` | int | `8` | Addon pull 控制器最大并发调和数 | | `--task-deployment-push-controller-workers` | int | `8` | Task Deployment push 控制器最大并发调和数 | -| `--task-daemonset-push-controller-workers` | int | `8` | Task DaemonSet push 控制器最大并发调和数 | | `--task-statefulset-push-controller-workers` | int | `8` | Task StatefulSet push 控制器最大并发调和数 | | `--node-push-controller-workers` | int | `8` | Node push 控制器最大并发调和数 | | `--pod-push-controller-workers` | int | `8` | Pod push 控制器最大并发调和数 | diff --git a/apps/rlark/docs/zh/reference/crd.md b/apps/rlark/docs/zh/reference/crd.md index 08a39d5..3d4d5a2 100644 --- a/apps/rlark/docs/zh/reference/crd.md +++ b/apps/rlark/docs/zh/reference/crd.md @@ -22,7 +22,7 @@ 底层 CRD API 遵循 Kubernetes 资源语义，可包含列表、创建、读取、替换、Patch、删除、集合删除及 status 子资源等操作。是否支持某个字段或操作，以当前 CRD manifest 和生成结果为准。 -状态阶段枚举中，`Pod.status.phase` 支持 `Pending`、`Running`、`Succeeded`、`Failed`、`Unknown`；`Node.status.phase` 支持 `Online`、`Offline`；`Cluster.status.phase` 支持 `Online`、`Offline`、`Downgraded`、`Cordoned`、`Draining`；`Node.spec.agentType` 支持 `Kubernetes`、`Docker`、`Raw`。 +状态阶段枚举中，`Pod.status.phase` 支持 `Pending`、`Running`、`Succeeded`、`Failed`、`Unknown`；`Node.status.phase` 支持 `Online`、`Offline`；`Cluster.status.phase` 支持 `Online`、`Offline`、`Downgraded`、`Cordoned`、`Draining`、`Delisted`；`Node.spec.agentType` 支持 `Kubernetes`、`Docker`、`Raw`。 Gateway 不会原样透传这套完整 API：它实际只开放部分资源和方法，且没有 collection delete（集合删除）或 `/status` 路由；因此不能根据本页推断 Gateway 存在同名接口。Gateway 的准确路由边界请查阅 [Gateway API 参考](../api/reference.md)，完整字段和底层操作请查阅 [英文生成参考](../../reference/crd.md)。 diff --git a/apps/rlark/docs/zh/storage-api.md b/apps/rlark/docs/zh/storage-api.md index fc0ff19..d547e20 100644 --- a/apps/rlark/docs/zh/storage-api.md +++ b/apps/rlark/docs/zh/storage-api.md @@ -20,6 +20,7 @@ Storage API 用于管理跨多个集群的 RLark StorageClass 声明。Gateway "data": { "ceph-rbd": { "name": "ceph-rbd", + "storageClassName": "sc-0123456789abcdef", "clusters": ["agent-beijing", "agent-shanghai"], "description": "Ceph RBD StorageClass", "bucket": "", @@ -36,7 +37,8 @@ Storage API 用于管理跨多个集群的 RLark StorageClass 声明。Gateway | 字段 | 类型 | 说明 | |------|------|------| -| `name` | `string` | StorageClass 名称 | +| `name` | `string` | 供人阅读的 StorageClass 声明名称 | +| `storageClassName` | `string` | 已下发到目标数据面集群的生成式 `sc-` Kubernetes StorageClass 名称；PVC 的 `spec.storageClassName` 必须使用该值 | | `clusters` | `[]string` | 该 StorageClass 已配置的目标集群 ID 列表 | | `description` | `string` | 可读描述 | | `bucket` | `string` | 存储桶名称 | @@ -44,12 +46,12 @@ Storage API 用于管理跨多个集群的 RLark StorageClass 声明。Gateway | `endpoint` | `string` | 存储服务端点地址 | | `region` | `string` | 存储区域 | | `pathStyle` | `bool` | 是否使用 path-style 寻址 | -| `accessKeyId` | `string` | Access Key ID；不会返回 Access Key Secret | +| `accessKeyId` | `string` | 已掩码的 Access Key ID（`<前两个字符>*****<后两个字符>`）；不会返回 Access Key Secret | ### 2. 获取存储提供商列表 **GET** `/api/v1/storage/storageclass/provider` -列出支持的存储提供商列表（AWS S3、阿里云 OSS、MinIO、Ceph 等共 31 个提供商）。 +列出支持的存储提供商。所有已认证用户均可读取该目录，以及创建、更新和删除 StorageClass。该目录是创建、更新 StorageClass 时 `provider` 合法取值的唯一来源：提交 `value`，`name` 仅用于展示。rclone 遇到无法识别的 provider 不会报错，只会打印告警并回退到通用配置，因此 Gateway 会直接拒绝不在该目录中的 `provider`。 #### 响应示例 ```json @@ -57,7 +59,7 @@ Storage API 用于管理跨多个集群的 RLark StorageClass 声明。Gateway "data": [ { "name": "AWS S3", "value": "AWS" }, { "name": "阿里云 OSS", "value": "Alibaba" }, - { "name": "MinIO", "value": "MinIO" } + { "name": "MinIO", "value": "Minio" } ], "success": true } @@ -68,10 +70,32 @@ Storage API 用于管理跨多个集群的 RLark StorageClass 声明。Gateway 在控制面创建 StorageClass 声明。Replication 和 Delivery 控制器会将生成的 rclone CSI StorageClass 及其 Secret 异步分发到请求指定的目标集群。 +Gateway 在存储前会对 S3 配置做规范化和校验： + +- `provider` 必须命中提供商目录。canonical 取值、展示名以及旧客户端写入的历史拼写都会被接受，但记录中始终保存 canonical 取值；无法识别的取值返回 `400`。 +- `endpoint` 必须是不含存储桶名的 `http`/`https` URL，缺少 scheme 时默认补 `https`。存储桶名绝不会拼进 endpoint，因为 Gateway 的 S3 客户端和 rclone 都会自行拼接 virtual-hosted URL。 +- 只有发布了区域 endpoint 格式的提供商（如阿里云 OSS）才允许 `endpoint` 留空，此时必须提供 `region`，endpoint 由 region 推导得出。 +- 对仅支持 virtual-hosted 寻址的提供商（如阿里云 OSS、腾讯云 COS、华为云 OBS），`path_style` 会被强制置为 `false`；其他提供商保留提交的取值。 +- 完成结构校验后，Gateway 会在有界超时时间内发起带认证的 `HeadBucket` 请求。若配置的凭据无法访问存储桶，创建和更新请求会返回 `400 INVALID_REQUEST`；响应不会包含提供商错误详情或任何凭据。 + +该校验上线前写入的历史记录会在读取时做 canonical 化，确保 Gateway 的 S3 客户端与下发到数据面的 rclone 配置在 provider 和寻址方式上保持一致。 + +校验失败会使用稳定的机器可读错误码和不含敏感信息的结构化详情。缺少必填字段时，接口返回 `400`、错误码 `STORAGE_CLASS_REQUIRED_FIELDS` 以及 `details.fields`，例如： + +```json +{ + "code": "STORAGE_CLASS_REQUIRED_FIELDS", + "message": "bucket are required", + "details": { "fields": ["bucket"] } +} +``` + +客户端必须使用 `code` 和 `details.fields` 生成本地化字段提示，而不是翻译面向人类的 `message` 文本。 + ### 4. 更新 StorageClass **PUT** `/api/v1/storage/storageclass/{name}` -更新指定 StorageClass 声明的对象存储配置和目标集群。请求中的 `clusters` 是期望的最终目标集群集合；分发流程会从未包含的集群清理资源。更新时 `access_key_secret` 可留空，Gateway 会复用已有密钥。 +更新指定 StorageClass 声明的对象存储配置和目标集群。请求中的 `clusters` 是期望的最终目标集群集合；分发流程会从未包含的集群清理资源。更新时 `access_key_id` 和 `access_key_secret` 均可留空；Gateway 会读取并复用相应的已存储值，不会将凭据替换为空值。列表返回的 `accessKeyId` 使用五个星号掩码；客户端提交的 `access_key_id` 只要包含 `*****`，Gateway 就会将其视为未变更，而不是替换值。保留凭据时，Gateway 会使用之前存储的 Access Key ID 和 Secret 校验更新后的连接。 ### 5. 删除 StorageClass **DELETE** `/api/v1/storage/storageclass/{name}` @@ -79,13 +103,21 @@ Storage API 用于管理跨多个集群的 RLark StorageClass 声明。Gateway 删除 StorageClass 声明。分发 finalizer 会异步移除其 Delivery 声明，以及所有关联集群中的生成 StorageClass 和 Secret。 ### 6. 列出存储桶文件 -**GET** `/api/v1/storage/storageclass/{name}/{cluster}/list` +**GET** `/api/v1/storage/storageclass/{name}/list` -列出指定集群中 StorageClass 存储桶下的文件列表。 +列出 StorageClass 存储桶中的文件。Gateway 会直接从控制面 StorageClass 声明读取存储桶。 #### 路径参数 - `name`：StorageClass 名称 -- `cluster`：集群 ID（如 `agent-beijing`） + +#### 查询参数 +- `prefix`（可选）：要列出的对象键前缀。非根目录前缀使用 `/` 分隔且以 `/` 结尾，例如 `a/b/c/`。 +- `delimiter`（可选）：对象键分隔符。默认值为 `/`；使用该分隔符时，直属子目录会在 `common_prefixes` 中返回。 +- `maxKeys`（可选）：返回的对象和公共前缀的最大数量。 + +#### 目录导航 + +对象存储使用前缀而不是真实目录。文件浏览器将非空目录前缀视为以 `/` 结尾的 canonical 值。例如，目录路径 `a/b/c` 表示为 `a/b/c/`；其上级目录为 `a/b/`，面包屑目标依次为 `a/`、`a/b/` 和 `a/b/c/`。即使某个前缀不包含直属对象或子前缀，上述导航仍然可用。 #### 响应示例 ```json @@ -99,13 +131,12 @@ Storage API 用于管理跨多个集群的 RLark StorageClass 声明。Gateway ``` ### 7. 上传文件 -**POST** `/api/v1/storage/storageclass/{name}/{cluster}/upload` +**POST** `/api/v1/storage/storageclass/{name}/upload` 向指定 StorageClass 存储桶上传文件，使用 multipart/form-data 格式。 #### 路径参数 - `name`：StorageClass 名称 -- `cluster`：集群 ID #### 请求体 multipart/form-data，字段 `file` 为上传的文件。 @@ -119,13 +150,12 @@ multipart/form-data，字段 `file` 为上传的文件。 ``` ### 8. 下载文件 -**GET** `/api/v1/storage/storageclass/{name}/{cluster}/object/*key` +**GET** `/api/v1/storage/storageclass/{name}/object/*key` 下载指定存储桶中的对象，返回原始文件内容。 #### 路径参数 - `name`：StorageClass 名称 -- `cluster`：集群 ID - `key`：对象路径（如 `model-checkpoint.pt` 或 `logs/training.log`） #### 响应 @@ -133,13 +163,12 @@ multipart/form-data，字段 `file` 为上传的文件。 - `404`：文件不存在 ### 9. 删除文件 -**DELETE** `/api/v1/storage/storageclass/{name}/{cluster}/object/*key` +**DELETE** `/api/v1/storage/storageclass/{name}/object/*key` 删除指定存储桶中的对象。 #### 路径参数 - `name`：StorageClass 名称 -- `cluster`：集群 ID - `key`：对象路径 #### 响应示例 diff --git a/apps/rlark/docs/zh/user-guide/clusters-nodes.md b/apps/rlark/docs/zh/user-guide/clusters-nodes.md index 14318c6..c5a7b4c 100644 --- a/apps/rlark/docs/zh/user-guide/clusters-nodes.md +++ b/apps/rlark/docs/zh/user-guide/clusters-nodes.md @@ -9,13 +9,22 @@ 3. 检查集群状态、在线率和 Worker 数量。 4. 打开计划使用的集群。 -Cluster 是集群作用域的 RLark 资源。其 cluster Agent 每 15 秒续约一次控制面的 `heartbeat` Lease，Lease 在 45 秒后过期。每个已注册 Worker 还需要由 `rlark-agent-node` 维护自身的 `node-heartbeat-` Lease；节点 Lease 缺失或过期时，仅该 Worker 会离线。各状态含义如下： +Cluster 是集群作用域的 RLark 资源。`spec.agentNamespace` 是其 Node CR 成员关系的权威边界：集群详情与节点数只包含该命名空间中的 Node CR。`rlark.io/cluster-id` 仅用于兼容查询，不能证明成员归属。其 cluster Agent 每 15 秒续约一次控制面的 `heartbeat` Lease，Lease 在 45 秒后过期。每个已注册 Worker 还需要由 `rlark-agent-node` 维护自身的 `node-heartbeat-` Lease；节点 Lease 缺失或过期时，仅该 Worker 会离线。 + +列表的**状态**列和状态筛选使用以下集群阶段： - **Online（在线）**：集群心跳有效，且所有已注册 Worker 在线。 - **Offline（离线）**：心跳 Lease 缺失或已过期。 -- **Downgraded（降级）**：集群心跳有效，但至少一个已注册 Worker 离线。 -- **Cordoned（已封锁）**：持续的 Cluster 意图禁止调度。 -- **Draining（排空中）**：Cluster 处于持续的排空期望状态。 +- **Downgraded（服务降级）**：集群心跳有效，但至少一个已注册 Worker 离线。 +- **Cordoned（停用调度）**：持续的 Cluster 意图禁止调度；该操作不会停止已有 Task。 +- **Draining（排空中）**：管理员已请求有序排空。RLark 会将已处于 Succeeded 或 Failed 终态的 Task 视为已排空，并为其余 Task 记录排空停止请求，随后等待每个 Task 针对当前 generation 上报停止状态。拥有 Job 的 Task 由 Job 控制器下发暂停；独立 Task 由 Cluster 控制器直接暂停。检查会从 1 秒到 1 分钟指数退避。排空超过 30 分钟仍未完成时，会保持请求状态，并通过 `Progressing=True`、`Reason=DrainTimedOut` Condition 列出阻塞的 Task 以便恢复处理；排空完成本身不会更改持续的排空或停用调度意图。出现当前 generation 的 `Progressing=False`、`Reason=DrainComplete` 后，可使用下架隐藏集群，使用 Undrain 恢复开始排空前保存的停用调度状态，或使用故障恢复取消排空意图并恢复调度。取消排空会清除停止请求，并且只恢复由排空暂停的独立 Task；排空前就已暂停的 Task 保持自身的期望状态。 +- **Delisted（已下架）**：管理员已将集群及其节点从用户控制台隐藏。对非管理员，节点页面和 Node 列表 API 不返回该集群的节点；重新上架后节点会再次可见。管理员仍可查看集群及其节点，以便执行运维或上架恢复。 + +列表还显示 Worker 总数、在线数和离线数，并按 `在线 Worker / Worker 总数` 计算在线率。数量和在线率反映 Worker 心跳观测结果，本身不会改变集群阶段或调度资格。 + +管理员可使用**故障维修（Cordon）**暂停新任务调度；当集群处于 Cordoned 状态或其排空已完成时，使用**故障恢复（Uncordon）**恢复新任务调度；使用**驱逐任务（Drain）**仅停止所选集群中的 RLark Task，等待排空完成后使用**下架（Delist）**停止向用户展示集群，使用**上架（Restore）**恢复可见性且不改变此前的调度意图。下架与由心跳缺席导致的 **Offline（离线）**状态不同。仍处于排空状态的集群，只有当前 generation 的 Condition 为 `Progressing=False`、`Reason=DrainComplete` 后才能下架或执行故障恢复；排空未完成时，这些操作会被拒绝。对已完成排空的集群执行故障恢复时，还会清除 `spec.draining` 和保存的停用调度状态，使集群重新可被调度，而不是继续被排空意图阻塞。故障恢复不会修复 Agent 连接或节点健康状态；清除暂停调度意图后，集群状态会根据已观测的心跳和节点状态重新计算。已下架集群需先上架后才能执行故障恢复。Job 的结果仍由已观测的 Task 状态聚合得出。 + +创建 Job 时，控制台展示 **Online（在线）** 和 **Downgraded（服务降级）** 集群。Job 控制器在解析 Task 的节点选择器时也会在服务端强制执行该规则：仅接受在线且可调度的 Node，且其所属的唯一 Cluster 必须同时满足 `spec.cordoned`、`spec.draining`、`spec.delisted` 均为 false，并且阶段为 **Online（在线）** 或 **Downgraded（服务降级）**。Cordoned、Draining、Offline 和 Delisted 集群均不会作为候选，通过 API 创建的 Job 同样受此约束。Downgraded 集群只会将任务调度至其中在线且可调度的 Worker；离线 Worker 仍不具备调度资格。Gateway 会拒绝在 Cluster 处于 Cordoned、Draining 或 Delisted 期间向其 agent namespace 创建新的独立 Task，但允许更新和 patch 已有 Task 以进行运维恢复。Cordon 不会停止已有 Task；已有 RLark Task 仍由 Drain 负责停止。 > **截图说明：** 截图来自示例环境，资源名称和数据仅供说明，实际环境会有所不同。 diff --git a/apps/rlark/docs/zh/user-guide/console.md b/apps/rlark/docs/zh/user-guide/console.md index e5b0844..ab7be65 100644 --- a/apps/rlark/docs/zh/user-guide/console.md +++ b/apps/rlark/docs/zh/user-guide/console.md @@ -25,7 +25,7 @@ RLark 有两个登录入口： | 页面 | 用途 | | -------- | ------------------------------------------ | -| 总览 | 集群、节点、机器人和运行中任务的仪表盘汇总 | +| 总览 | 集群、可调度工作负载节点、机器人和运行中任务的仪表盘汇总 | | 集群 | 浏览和检查已纳管的数据面集群 | | 节点 | 筛选和检查节点资源、调度状态和健康状态 | | 任务 | 创建、监控和管理训练任务 | @@ -56,6 +56,36 @@ RLark 有两个登录入口： 点击**刷新**可在不离开当前页面的情况下获取最新数据。请求进行时，受影响的表格或仪表盘会变淡并暂时禁用交互，同时在区域中央显示进度指示；已有数据行会保留到新响应返回。总览、集群与节点、域、任务与 Worker、存储与文件、SSH 公钥、镜像仓库和系统配置均使用这一套一致的反馈方式。 +### 界面语言 + +顶部的语言开关对中文和英文同时生效。除静态文案外，它还会本地化控制台从控制面状态推导出的取值：节点状态原因、Addon 阶段、CPU 核数单位、Worker 筛选中推导出的节点类型，以及列表分隔符。登录页和 WebTerminal 页同样遵循该开关。从数据面原样透传的取值——节点标签、操作系统与架构标识符等服务端提供的字符串——按上报内容展示，不做翻译。 + +### 节点表格中的长取值 + +在节点详情的 Worker 表格中，过长的实例名称会省略号截断；展开的 Pod 子表中，网络域同样会截断显示。将鼠标悬停在被截断的取值上即可在提示中查看完整内容。这样无需水平滚动也能看到状态列。 + +### 按标签筛选任务 + +任务列表的标签筛选与 Gateway 的语义一致：不同标签 key 之间为「且」，同一 key 下的多个值之间为「或」。只有同时携带所有被选中的 key，且每个 key 至少命中一个被选中的值的任务才会出现在列表中。因此，选中一组不会同时出现在同一个任务上的 key 时会得到空列表，这是筛选语义本身，而不是请求失败。弹层会在两列上方说明该规则，**全选/取消**只作用于当前聚焦 key 的值列表，不会作用于 key 列。 + +点击 key 所在的行即可聚焦该 key，右栏会一直展示它的值，直到点击另一个 key。悬停不会改变聚焦，因此指针移向值列表的途中不会换掉右栏内容。key 前面的勾选框是独立的控件，用于一次性选中或清空该 key 的所有值。 + +### 任务列表中的角色数量 + +**角色数量**列统计任务去重后的角色名数量，与任务详情页和任务编辑器展示的角色名一致。它不来自 `Task.spec.role`：后者只是用于分组展示的粗粒度提示（`Actor`、`Rollout`、`Env`），会把多个业务角色折叠成同一个取值。 + +### 复制按钮 + +复制按钮不会静默完成，而是就地给出结果反馈：图标切换为对勾约 1.6 秒后恢复为复制图标。任务详情页角色运行配置中的镜像复制按钮还会泛起一次绿色脉冲，并把提示文案切换为**已复制**。按钮没有变化说明复制未发生，通常是浏览器阻止了剪贴板访问。 + +控制台中所有复制按钮共用同一条剪贴板实现。浏览器只在安全上下文（HTTPS 或 `localhost`）下提供异步剪贴板接口，因此控制台通过普通 HTTP 访问时，复制会自动回退到兼容方式而不是直接失效。该行为覆盖 SSH 公钥、SSH 连接命令、Worker SSH 命令、API 示例、部署配置 YAML、工作流 YAML 以及任务日志的复制按钮。只有复制确实成功后才会给出成功反馈。 + +### 关闭表单弹层 + +需要填写内容的弹层——创建、复制、编辑或重启编辑任务，创建工作流，创建或编辑存储类，添加或编辑镜像仓库凭据，添加 SSH 公钥，创建网络域，以及编辑任务标签——在点击弹层外的变暗区域时不会关闭。由于控制台无法恢复已丢弃的输入，这些弹层只能通过右上角的关闭按钮或**取消**按钮退出；任务、工作流、存储类和镜像仓库弹层还支持 `Esc` 关闭。 + +不收集输入的确认弹窗（例如任务生命周期操作、删除任务、重启方式选择和集群生命周期操作）仍可通过点击弹窗外部关闭。 + ## API 等效操作 已支持的资源操作可通过 [Gateway API](../api/reference.md) 完成。独立 Gateway 默认使用 `http://:8080`；`rlarkadm` 部署使用其配置的 Service 与 UI 代理。并非每个 UI 交互都有一一对应的公开接口，请以 API 参考为准。 diff --git a/apps/rlark/docs/zh/user-guide/jobs.md b/apps/rlark/docs/zh/user-guide/jobs.md index de47723..ef5827b 100644 --- a/apps/rlark/docs/zh/user-guide/jobs.md +++ b/apps/rlark/docs/zh/user-guide/jobs.md @@ -14,9 +14,10 @@ Job 是面向用户的工作负载。创建任务时选择模板或配置 Task 2. 从规格列表中单选 GPU 或具身设备型号；列表会同时展示设备和节点的可用量/总量。选定后填写两种调度方式共用的单 Worker 资源数量。调试任务可以填写 `0`，此时保留所选型号的节点范围，但不申请对应设备。 3. 选择一种调度方式： - **自动选择**：填写需要创建的 Worker 数量，控制台会按可调度容量校验总申请量并自动选择符合条件的节点。 - - **指定节点**：单击符合条件的节点或拖拽框选，每个选中节点创建一个 Worker；再次单击或框选已选节点可取消选择。 + - **指定节点**：单击符合条件的节点或拖拽框选，每个选中节点创建一个 Worker；再次单击或框选已选节点可取消选择。之后修改单 Worker 资源数量时，空闲容量仍然满足新数量的已选节点会保留，只有不再满足的节点会被移除。 +4. 检查统一的资源总结，然后配置镜像、准备脚本、环境变量和存储挂载。 -复制任务时会保留原任务的调度方式。未指定具体节点的角色仍使用自动选择，不会被转换为指定节点。4. 检查统一的资源总结，然后配置镜像、准备脚本、环境变量和存储挂载。 +复制任务时会保留原任务的调度方式。未指定具体节点的角色仍使用自动选择，不会被转换为指定节点。 提交任务后，在任务详情页确认状态进入运行中并查看 Worker。 @@ -24,6 +25,13 @@ Job 是面向用户的工作负载。创建任务时选择模板或配置 Task Worker 处于等待中时，可悬停或聚焦其状态旁的信息图标。RLark 会查询该 Worker 对应的数据面 Pod、Task 或节点事件，并按当前界面语言将已识别的 Kubernetes 事件归类为用户可读的等待原因。映射覆盖常见的调度、节点压力与可用性、镜像拉取、存储卷、运行环境、容器创建及健康检查问题；未识别事件显示通用等待提示，不直接展示底层原始消息。镜像拉取进度仍会单独展示，且只有运行时实际上报字节进度时才显示百分比。 +提示框展示的是当前的阻塞原因，而不是原始事件流水： + +- `Pulling` 与 `Pulled` 是**按容器**上报的时点事件，Worker Pod 还会拉取平台注入的 sidecar 镜像。因此当前原因中只保留最新的一条拉取进展，更早的归入 **历史事件**；一旦镜像拉取失败，或存在不早于该进展的 Warning 事件，则所有拉取进展都会归入历史事件，避免 sidecar 拉取成功的"镜像已拉取"与 Worker 失败提示并列出现。 +- kubelet 对镜像拉取失败和容器启动失败都上报 `Failed`。RLark 会按事件消息进一步收敛，使镜像不可用时显示"镜像地址无效"或"镜像拉取失败"，而不是"Worker 启动失败"。 +- 同一原因的重复事件会按原因合并，保留最近一次上报的时间，并整体按时间倒序排列。**历史事件** 分区以灰色删除线保留被取代的条目，便于排查。 + + ## 任务类型 RLark 支持以下任务类型，每种类型预配置了适合该工作负载的默认角色： @@ -53,10 +61,12 @@ RLark 支持以下任务类型，每种类型预配置了适合该工作负载 - **准备脚本** — 在主入口命令之前运行的命令。用于环境设置、依赖安装或数据准备。 -- **存储挂载** — 向 Worker 容器挂载存储。支持两种类型： +- **存储挂载** — 向 Worker 容器挂载存储。任务编辑器可创建两种类型： - **hostPath**：挂载宿主机节点上的目录，任务生命周期操作不会删除其中的数据。 - **PVC**（PersistentVolumeClaim）：使用所选存储类挂载 Kubernetes 存储。当前 Web UI 写入 `ephemeral.volumeClaimTemplate`，因此这些 PVC 属于 Pod 级资源，会随 Pod 删除并重新创建。导入或较早创建、使用 `persistentVolumeClaim` 和已废弃 PVC map 的 Task 则遵循保留型兼容生命周期。 + 通过 API 提交的任务还可能使用其他 Kubernetes 原生卷类型，例如 `emptyDir`、`configMap`、`secret`。任务编辑器无法创建或修改这类卷，但会在角色的挂载列表和任务详情页以只读方式展示，并在编辑或复制任务时原样保留其原始定义；编辑器对这类挂载唯一支持的改动是删除。 + > **截图说明：** 截图来自示例环境，资源名称和数据仅供说明，实际环境会有所不同。 ![Worker 配置](../../images/ui/create-job-worker-configuration.png) @@ -65,13 +75,13 @@ RLark 支持以下任务类型，每种类型预配置了适合该工作负载 配置适用于任务中所有 Worker 的设置： -- **Header 角色** — 选择一个角色作为 Header。Ray 任务中的 Header 角色必须恰好配置 1 个 Worker；它负责协调分布式训练，其 IP 地址会通知所有其他 Worker。 +- **Header 角色** — 选择一个角色作为 Header。Ray 任务中的 Header 角色必须恰好配置 1 个 Worker；它负责协调分布式训练，其 IP 地址会通知所有其他 Worker。副本数大于 1 的角色在该选择器中不可选；若之后把已选 Header 角色的副本数改大，控制台会回退到第一个单 Worker 角色，保证向导仍可修正。 - **跨集群网络域** — 如果后台配置了网络域，控制台会按名称选择第一个网络域，并为所有任务自动启用，不受 Worker 是否跨集群影响；未配置时不会写入网络域。 - **SSH 公钥** — 选择写入 Job workload SSH 配置的公钥。内置 Ray-on-Kubernetes workload 会通过 Task-owned Secret 交付，挂载到每个 Worker main container 的 `/var/run/rlark/ssh/authorized_key`，并启动 optional `rlark-tools sshd` 服务。SSH 故障不会使 Task 失败；其他 workload implementation 不会仅因设置此字段而自动获得 SSH 能力。 -- **运行命令** — 主训练脚本或入口命令。在所有准备脚本完成后执行。 +- **运行命令** — 主训练脚本或入口命令。在所有准备脚本完成后执行。Header 节点执行完运行命令后不会退出，而是继续阻塞保持 Ray 集群运行，直到任务被手动停止或删除。 - **TensorBoard** — 为任务启用 TensorBoard 监控。配置日志目录路径和 TensorBoard 服务端口。 @@ -159,7 +169,7 @@ nvidia-smi # 检查 GPU 状态（如有 GPU 分配） ### 文件传输 -Worker 列表中的钥匙按钮用于复制 SSH 连接命令。该功能依赖管理员配置 SSH 跳板地址；未配置时按钮会禁用并提示原因，避免复制空内容。 +Worker 列表中的钥匙按钮用于复制 SSH 连接命令。该功能依赖管理员配置 SSH 跳板地址；未配置时按钮会禁用并提示原因，避免复制空内容。复制命令统一通过节点 sshd 的 2222 端口转发到目标 Pod。在非 HTTPS 环境下，复制功能自动降级使用兼容方案，不受浏览器 Clipboard API 限制。 WebTerminal 支持文件上传和下载： @@ -213,6 +223,10 @@ WebTerminal 支持文件上传和下载： - **编辑** — 修改已停止任务的配置（角色、资源、命令）并重新提交。 - **克隆** — 复制现有任务配置，作为新任务的模板。 +两种操作都会用源任务预填编辑器，并保留源任务的角色，包括不属于所选任务类型默认角色的自定义角色。每个角色会恢复容器镜像、运行命令、准备脚本、环境变量、副本数、CPU、内存和设备申请、调度方式以及存储挂载；编辑器无法编辑的卷类型按原始定义保留。预填后切换任务类型只会更新任务类型标签，不会改动角色列表：角色是用户已经确认过的配置，需要新类型的默认角色时请手动添加。 + +Job 本身不保存源集群信息，编辑器会根据各角色的节点选择器反查集群，因此重新提交前请确认集群选择是否正确。 + ## 提交前检查清单 提交任务前请确认： diff --git a/apps/rlark/docs/zh/user-guide/ssh-keys.md b/apps/rlark/docs/zh/user-guide/ssh-keys.md index e749d95..e5c0bd4 100644 --- a/apps/rlark/docs/zh/user-guide/ssh-keys.md +++ b/apps/rlark/docs/zh/user-guide/ssh-keys.md @@ -16,6 +16,8 @@ RLark 会校验公钥格式，并要求 SSH 用户名和公钥内容在全局范围内均不可重复。任一内容已存在时，表单会在上传前显示行内错误提示。当 Server SSH 地址可用时，页面还会显示配置好的跳板连接命令。 +连接命令和列表中每条公钥右侧都提供复制按钮。复制成功后按钮会短暂变为绿色对勾，与控制台其他复制按钮一致；复制失败时按钮不会给出成功提示。控制台通过 HTTP 访问（非安全上下文）时浏览器不提供异步剪贴板接口，此时复制会自动回退到兼容方式，无需改用 HTTPS。 + ## 任务二：为 Job 选择密钥 1. 打开**任务**并开始创建 Job。 @@ -34,14 +36,18 @@ RLark 会校验公钥格式，并要求 SSH 用户名和公钥内容在全局范 每条选中的公钥会写入适用 Task 模板的 `rlinf.io/ssh/v1` Environment，供 workload 交付。只在 SSH 公钥页面登记或修改记录不会改变已有 Job 或 Task。Job 详情中的多条已配置公钥会通过可滚动列表展示。 +由于 Job 保存的是公钥原文而不是对密钥记录的引用，公钥的生命周期可能长于其用户记录。此时 Job 详情不再显示用户名，而是标记为`用户已删除`，并通过提示说明该用户记录已从 SSH 公钥管理中删除、公钥仍注入在运行中的 Pod 内。复制、编辑或编辑后重启 Job 时，这类公钥会从草稿中移除，因为选择列表无法展示也无法取消勾选没有用户记录的公钥；表单会提示移除了多少条公钥，便于在提交前补充替换密钥。 + 对于内置 Ray-on-Kubernetes workload，Agent 会把 Task 选中的公钥存入归当前 Task incarnation 所有的 Secret，将其 `authorized_key` item 以只读方式挂载到每个 main container 的 `/var/run/rlark/ssh/authorized_key`，并将 `rlark-tools sshd` 作为 optional supervised service 启动。该 mount 不使用 `subPath`，`sshd` 每次认证都会重新打开该文件。空文件表示关闭 workload key 认证。SSH 服务故障可能导致 Pod 无法通过 SSH 访问，但不会停止 Task。Docker、Raw 或任意自定义 workload implementation 不具备这一自动服务生命周期。 +Job Controller 为所有 Task 统一设置 SSH 端口 2222。前端复制的 SSH 命令会通过节点 sshd 的 2222 端口转发到目标 Pod。 + ## 任务三：通过堡垒机连接 管理员配置 Server SSH 入口后，使用控制台显示的命令，或从 Job/节点详情复制指定 Worker 的命令。典型命令如下： ```bash -ssh -J : root@ +ssh -J : root@ -p 2222 ``` 外层 SSH 用户名必须与公钥登记的用户名一致。目标容器必须提供 SSH 服务，并接受所选公钥。 diff --git a/apps/rlark/docs/zh/user-guide/storage.md b/apps/rlark/docs/zh/user-guide/storage.md index 1155803..428b6b6 100644 --- a/apps/rlark/docs/zh/user-guide/storage.md +++ b/apps/rlark/docs/zh/user-guide/storage.md @@ -2,13 +2,14 @@ ## 存储类型 -Job 编辑器提供两种存储选择：主机目录和 PVC 存储挂载。其生命周期由 Kubernetes 原生 PodTemplate volume 形态决定： +Job 编辑器可创建两种存储：主机目录和 PVC 存储挂载。通过 API 提交的 Job 还可以使用其他 Kubernetes 原生 PodTemplate volume，编辑器对这类卷只做展示。其生命周期由 Kubernetes 原生 PodTemplate volume 形态决定： | 类型 | 适用场景 | 生命周期 | |------|----------|----------| | 主机目录 | 数据已在节点上，高 I/O | 任务生命周期操作不删除数据 | | 对象存储（Pod 级临时 PVC） | 当前 Web UI 为 PVC 挂载写入的形态 | Kubernetes 随 Pod 创建和删除 PVC | | 已有持久 PVC 引用 | Task volume 使用 `persistentVolumeClaim.claimName` | RLark 不创建或删除该 PVC；生命周期由管理员管理 | +| 其他 Kubernetes 原生卷（`emptyDir`、`configMap`、`secret` 等） | 通过 API 提交，Job 编辑器仅展示 | 由卷类型本身决定；编辑或复制任务时 RLark 原样写回其原始定义 | ## 主机目录 @@ -56,6 +57,7 @@ Job 编辑器提供两种存储选择：主机目录和 PVC 存储挂载。其 - 已有 `persistentVolumeClaim`：RLark 不创建、收养、修改或删除引用的 PVC；其生命周期和数据保留不属于 Task 管理范围 - Pod 级 `ephemeral.volumeClaimTemplate`：删除 Worker Pod 时删除对应临时 PVC，新 Pod 获得新 PVC - hostPath：生命周期操作不删除节点数据 +- 其他 Kubernetes 原生卷：Job 编辑器无法创建或修改；编辑或复制任务时会原样写回其原始定义，因此生命周期仍由该卷类型本身决定 已有 PVC 引用必须填写 `claimName`，且该 claim 必须已存在于 workload namespace。挂载或绑定失败由 Kubernetes 通过 workload 和 Pod 状态报告。删除被引用的 PVC 属于外部管理操作；删除前应确认没有 workload 使用它，并考虑绑定 PV 的 reclaim policy。 diff --git a/apps/rlark/pkg/agent/cluster_agent.go b/apps/rlark/pkg/agent/cluster_agent.go index 5eed85d..e3838bb 100644 --- a/apps/rlark/pkg/agent/cluster_agent.go +++ b/apps/rlark/pkg/agent/cluster_agent.go @@ -120,7 +120,6 @@ func (c *clusterAgent) Run(ctx context.Context) error { taskController.PullMaxConcurrentReconciles = c.a.config.ControllerConcurrency.TaskPull taskController.PushMaxConcurrentReconciles = map[string]int{ "task-deployment": c.a.config.ControllerConcurrency.TaskDeployment, - "task-daemonset": c.a.config.ControllerConcurrency.TaskDaemonSet, "task-statefulset": c.a.config.ControllerConcurrency.TaskStatefulSet, } tc := task.NewTaskController(taskController) diff --git a/apps/rlark/pkg/agent/config.go b/apps/rlark/pkg/agent/config.go index d83bf8c..7dd5e83 100644 --- a/apps/rlark/pkg/agent/config.go +++ b/apps/rlark/pkg/agent/config.go @@ -20,7 +20,6 @@ type ControllerConcurrencyConfig struct { TaskPull int AddonPull int TaskDeployment int - TaskDaemonSet int TaskStatefulSet int NodePush int PodPush int @@ -31,7 +30,6 @@ func defaultControllerConcurrencyConfig() ControllerConcurrencyConfig { TaskPull: defaultControllerMaxConcurrentReconciles, AddonPull: defaultControllerMaxConcurrentReconciles, TaskDeployment: defaultControllerMaxConcurrentReconciles, - TaskDaemonSet: defaultControllerMaxConcurrentReconciles, TaskStatefulSet: defaultControllerMaxConcurrentReconciles, NodePush: defaultControllerMaxConcurrentReconciles, PodPush: defaultControllerMaxConcurrentReconciles, @@ -42,7 +40,6 @@ func (c *ControllerConcurrencyConfig) SetupFlags(fs *pflag.FlagSet) { fs.IntVar(&c.TaskPull, "task-pull-controller-workers", c.TaskPull, "Maximum concurrent Task pull reconciles") fs.IntVar(&c.AddonPull, "addon-pull-controller-workers", c.AddonPull, "Maximum concurrent Addon pull reconciles") fs.IntVar(&c.TaskDeployment, "task-deployment-push-controller-workers", c.TaskDeployment, "Maximum concurrent Task Deployment push reconciles") - fs.IntVar(&c.TaskDaemonSet, "task-daemonset-push-controller-workers", c.TaskDaemonSet, "Maximum concurrent Task DaemonSet push reconciles") fs.IntVar(&c.TaskStatefulSet, "task-statefulset-push-controller-workers", c.TaskStatefulSet, "Maximum concurrent Task StatefulSet push reconciles") fs.IntVar(&c.NodePush, "node-push-controller-workers", c.NodePush, "Maximum concurrent Node push reconciles") fs.IntVar(&c.PodPush, "pod-push-controller-workers", c.PodPush, "Maximum concurrent Pod push reconciles") @@ -53,7 +50,6 @@ func (c ControllerConcurrencyConfig) Validate() error { "task-pull": c.TaskPull, "addon-pull": c.AddonPull, "task-deployment-push": c.TaskDeployment, - "task-daemonset-push": c.TaskDaemonSet, "task-statefulset-push": c.TaskStatefulSet, "node-push": c.NodePush, "pod-push": c.PodPush, diff --git a/apps/rlark/pkg/agent/config_test.go b/apps/rlark/pkg/agent/config_test.go index 433d1bb..33e0ebc 100644 --- a/apps/rlark/pkg/agent/config_test.go +++ b/apps/rlark/pkg/agent/config_test.go @@ -11,7 +11,7 @@ import ( func TestDefaultControllerConcurrency(t *testing.T) { got := DefaultConfig().ControllerConcurrency if got.TaskPull != 8 || got.AddonPull != 8 || got.TaskDeployment != 8 || - got.TaskDaemonSet != 8 || got.TaskStatefulSet != 8 || got.NodePush != 8 || got.PodPush != 8 { + got.TaskStatefulSet != 8 || got.NodePush != 8 || got.PodPush != 8 { t.Fatalf("default controller concurrency = %+v, want all 8", got) } if err := got.Validate(); err != nil { @@ -71,16 +71,15 @@ func TestControllerConcurrencyFlags(t *testing.T) { "--task-pull-controller-workers=1", "--addon-pull-controller-workers=2", "--task-deployment-push-controller-workers=3", - "--task-daemonset-push-controller-workers=4", - "--task-statefulset-push-controller-workers=5", - "--node-push-controller-workers=6", - "--pod-push-controller-workers=7", + "--task-statefulset-push-controller-workers=4", + "--node-push-controller-workers=5", + "--pod-push-controller-workers=6", }); err != nil { t.Fatal(err) } want := ControllerConcurrencyConfig{ - TaskPull: 1, AddonPull: 2, TaskDeployment: 3, TaskDaemonSet: 4, - TaskStatefulSet: 5, NodePush: 6, PodPush: 7, + TaskPull: 1, AddonPull: 2, TaskDeployment: 3, + TaskStatefulSet: 4, NodePush: 5, PodPush: 6, } if config.ControllerConcurrency != want { t.Fatalf("controller concurrency = %+v, want %+v", config.ControllerConcurrency, want) diff --git a/apps/rlark/pkg/agent/controllers/task/api/types.go b/apps/rlark/pkg/agent/controllers/task/api/types.go index 7bb3b2e..467bb89 100644 --- a/apps/rlark/pkg/agent/controllers/task/api/types.go +++ b/apps/rlark/pkg/agent/controllers/task/api/types.go @@ -6,7 +6,6 @@ const ( WorkloadTypeRay = "rlinf.io/ray" WorkloadVersionRayV1 = "v1" CarrierTypeKubernetesDeployment = "rlinf.io/kubernetes-deployment" - CarrierTypeKubernetesDaemonSet = "rlinf.io/kubernetes-daemonset" CarrierTypeKubernetesStatefulSet = "rlinf.io/kubernetes-statefulset" CarrierVersionKubernetesV1 = "v1" EnvironmentTypeSSH = "rlinf.io/ssh" @@ -46,15 +45,13 @@ type KubernetesReplicatedCarrierParametersV1 struct { Replicas *int32 `json:"replicas,omitempty"` } -// KubernetesDaemonSetCarrierParametersV1 is intentionally empty because a -// DaemonSet derives its instance count from eligible nodes. -type KubernetesDaemonSetCarrierParametersV1 struct{} - // SSHEnvironmentParametersV1 是 rlinf.io/ssh/v1 的参数合同。 // AuthorizedKeys contains public authentication material. It may be copied to // the local execution snapshot, but not to portable Plans or remote transport. +// Port is non-sensitive and may pass through to the Plan-bound requirement. type SSHEnvironmentParametersV1 struct { AuthorizedKeys []string `json:"authorizedKeys,omitempty"` + Port int32 `json:"port,omitempty"` } // NetworkEnvironmentParametersV1 configures the virtual network joined by a diff --git a/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task.go b/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task.go index 6f2845a..cd411b4 100644 --- a/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task.go +++ b/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task.go @@ -4,6 +4,7 @@ package compatibility import ( + "encoding/json" "errors" "fmt" @@ -62,6 +63,12 @@ func DecodeTask(task *rlarkv1alpha1.Task) (coordination.TaskSnapshot, error) { requirementParameters := parameters if configured.Type == taskapi.EnvironmentTypeSSH { requirementParameters = nil + var ssh taskapi.SSHEnvironmentParametersV1 + if err := json.Unmarshal(configured.Parameters.Raw, &ssh); err == nil && ssh.Port != 0 { + requirementParameters, _ = json.Marshal(struct { + Port int32 `json:"port"` + }{Port: ssh.Port}) + } } requirements.Environments = append(requirements.Environments, executionplan.EnvironmentRequirement{ Name: configured.Type, Type: configured.Type, Version: configured.Version, Parameters: requirementParameters, diff --git a/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task_test.go b/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task_test.go index ea61931..e752b10 100644 --- a/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task_test.go +++ b/apps/rlark/pkg/agent/controllers/task/compatibility/kubernetes/task_test.go @@ -15,7 +15,6 @@ import ( func TestDecodeTaskSeparatesWorkloadAndCarrier(t *testing.T) { carriers := []string{ taskapi.CarrierTypeKubernetesDeployment, - taskapi.CarrierTypeKubernetesDaemonSet, taskapi.CarrierTypeKubernetesStatefulSet, } for _, carrier := range carriers { diff --git a/apps/rlark/pkg/agent/controllers/task/controller.go b/apps/rlark/pkg/agent/controllers/task/controller.go index b56501b..228990e 100644 --- a/apps/rlark/pkg/agent/controllers/task/controller.go +++ b/apps/rlark/pkg/agent/controllers/task/controller.go @@ -192,10 +192,6 @@ func (c *Controller) AsKubePushReconcilers() map[base.KubernetesResource]base.Ku Name: "task-deployment", Type: &appsv1.Deployment{}, }: &pushDeploymentReconciler{c: c}, - { - Name: "task-daemonset", - Type: &appsv1.DaemonSet{}, - }: &pushDaemonSetReconciler{c: c}, { Name: "task-statefulset", Type: &appsv1.StatefulSet{}, diff --git a/apps/rlark/pkg/agent/controllers/task/environment/kubernetes/ssh/ssh.go b/apps/rlark/pkg/agent/controllers/task/environment/kubernetes/ssh/ssh.go index ec9ff8f..4052e30 100644 --- a/apps/rlark/pkg/agent/controllers/task/environment/kubernetes/ssh/ssh.go +++ b/apps/rlark/pkg/agent/controllers/task/environment/kubernetes/ssh/ssh.go @@ -47,10 +47,6 @@ type profileValue struct { Port int32 `json:"port"` } -type requirementOptions struct { - Port int32 `json:"port"` -} - // Resolver authorizes the built-in workload SSH Secret profile. It resolves no // raw material and performs no management or Runtime I/O. type Resolver struct{ RuntimeKind agentruntime.Kind } @@ -84,9 +80,9 @@ func (r Resolver) Resolve(ctx context.Context, request environment.ResolveReques if err != nil { return environment.ResolvedEnvironment{}, err } - options := requirementOptions{Port: 22} + options := taskapi.SSHEnvironmentParametersV1{Port: 22} if len(requirement.Parameters) != 0 { - decoded, err := taskapi.DecodeParametersStrict[requirementOptions](apiextensionsv1.JSON{ + decoded, err := taskapi.DecodeParametersStrict[taskapi.SSHEnvironmentParametersV1](apiextensionsv1.JSON{ Raw: append([]byte(nil), requirement.Parameters...), }) if err != nil { diff --git a/apps/rlark/pkg/agent/controllers/task/execution_test.go b/apps/rlark/pkg/agent/controllers/task/execution_test.go index 35e98a8..9ef88f6 100644 --- a/apps/rlark/pkg/agent/controllers/task/execution_test.go +++ b/apps/rlark/pkg/agent/controllers/task/execution_test.go @@ -95,7 +95,6 @@ func TestKubernetesExecutionCoordinatorEvaluatesAllBuiltinWorkloads(t *testing.T typeID string }{ {name: "Deployment", typeID: taskapi.CarrierTypeKubernetesDeployment}, - {name: "DaemonSet", typeID: taskapi.CarrierTypeKubernetesDaemonSet}, {name: "StatefulSet", typeID: taskapi.CarrierTypeKubernetesStatefulSet}, } for _, carrier := range carriers { @@ -104,9 +103,6 @@ func TestKubernetesExecutionCoordinatorEvaluatesAllBuiltinWorkloads(t *testing.T task.Generation = 3 task.Spec.Workload.Carrier.Type = carrier.typeID task.Annotations[host.TaskClaimedCarrierTypeAnnotation] = carrier.typeID - if carrier.typeID == taskapi.CarrierTypeKubernetesDaemonSet { - task.Spec.Workload.Carrier.Parameters = apiextensionsv1.JSON{Raw: []byte(`{}`)} - } task.Spec.Workload.Template.Spec.Containers = []corev1.Container{{Name: "main", Image: "example:v1"}} snapshot, err := compatibility.DecodeTask(task) if err != nil { @@ -825,19 +821,6 @@ func TestCoordinatorAdoptsOriginMainRunningWorkloadsWithoutTemplateDrift(t *test }, } }}, - {name: "daemonset", kind: raykubernetes.CarrierDaemonSet, old: func(template corev1.PodTemplateSpec) client.Object { - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "task", - Namespace: tenant.DefaultNamespace, - UID: "workload-uid", - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "task"}}, - Template: template, - }, - } - }}, {name: "statefulset", kind: raykubernetes.CarrierStatefulSet, old: func(template corev1.PodTemplateSpec) client.Object { one := int32(1) return &appsv1.StatefulSet{ @@ -859,19 +842,14 @@ func TestCoordinatorAdoptsOriginMainRunningWorkloadsWithoutTemplateDrift(t *test scheme := executionScheme(t) task := claimedExecutionTask(rlarkv1alpha1.TaskPhaseRunning) carrierType := taskapi.CarrierTypeKubernetesDeployment - if test.kind == raykubernetes.CarrierDaemonSet { - carrierType = taskapi.CarrierTypeKubernetesDaemonSet - } if test.kind == raykubernetes.CarrierStatefulSet { carrierType = taskapi.CarrierTypeKubernetesStatefulSet } task.Spec.Workload.Carrier.Type = carrierType task.Annotations[host.TaskClaimedCarrierTypeAnnotation] = carrierType - if test.kind != raykubernetes.CarrierDaemonSet { - one := int32(1) - parameters, _ := json.Marshal(taskapi.KubernetesReplicatedCarrierParametersV1{Replicas: &one}) - task.Spec.Workload.Carrier.Parameters = apiextensionsv1.JSON{Raw: parameters} - } + one := int32(1) + parameters, _ := json.Marshal(taskapi.KubernetesReplicatedCarrierParametersV1{Replicas: &one}) + task.Spec.Workload.Carrier.Parameters = apiextensionsv1.JSON{Raw: parameters} task.Spec.Workload.Template = &corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "task"}}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "origin-main:v1"}}}} // This fixture is the shape produced before Coordinator: name/selector // ownership, no workload-level management annotations or UID selector. @@ -1367,12 +1345,6 @@ func readWorkloadTemplate(t *testing.T, kube client.Client, kind string) corev1. t.Fatal(err) } return object.Spec.Template - case raykubernetes.CarrierDaemonSet: - var object appsv1.DaemonSet - if err := kube.Get(context.Background(), key, &object); err != nil { - t.Fatal(err) - } - return object.Spec.Template default: var object appsv1.StatefulSet if err := kube.Get(context.Background(), key, &object); err != nil { diff --git a/apps/rlark/pkg/agent/controllers/task/push_daemonset.go b/apps/rlark/pkg/agent/controllers/task/push_daemonset.go deleted file mode 100644 index 481096d..0000000 --- a/apps/rlark/pkg/agent/controllers/task/push_daemonset.go +++ /dev/null @@ -1,61 +0,0 @@ -package task - -import ( - "context" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1" - raykubernetes "github.com/rlinf/rlark/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes" -) - -// pushDaemonSetReconciler watches local DaemonSets and reports status to management Task. -type pushDaemonSetReconciler struct { - c *Controller -} - -// Reconcile reconciles the resource. -func (r *pushDaemonSetReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - logger := log.FromContext(ctx).WithValues("daemonset", req.NamespacedName) - - var ds appsv1.DaemonSet - if err := r.c.LocalKubeClient.Get(ctx, req.NamespacedName, &ds); err != nil { - if client.IgnoreNotFound(err) != nil { - return reconcile.Result{}, err - } - logger.V(1).Info("DaemonSet not found") - return reconcile.Result{}, nil - } - - taskName := ds.Annotations[ManagementTaskNameAnnotation] - taskNamespace := ds.Annotations[ManagementTaskNamespaceAnnotation] - if taskName == "" || taskNamespace == "" { - logger.V(1).Info("DaemonSet has no management-task annotation, skipping") - return reconcile.Result{}, nil - } - - var mgmtTask rlarkv1alpha1.Task - if err := r.c.ManagementClient.Get(ctx, types.NamespacedName{Name: taskName, Namespace: taskNamespace}, &mgmtTask); err != nil { - if client.IgnoreNotFound(err) != nil { - logger.Error(err, "failed to get management Task") - return reconcile.Result{}, err - } - logger.Info("management Task not found, skipping") - return reconcile.Result{}, nil - } - - if !pushOwnsTask(&mgmtTask, r.c.AgentType, &ds, raykubernetes.CarrierDaemonSet) { - logger.Info("DaemonSet no longer owns management Task status, skipping") - return reconcile.Result{}, nil - } - - if err := r.c.emitRuntimeTrigger(ctx, &mgmtTask, kubernetesWorkloadRef(&mgmtTask, &ds, raykubernetes.CarrierDaemonSet)); err != nil { - logger.Error(err, "failed to emit Task Runtime trigger") - return reconcile.Result{}, err - } - return reconcile.Result{}, nil -} diff --git a/apps/rlark/pkg/agent/controllers/task/push_deployment.go b/apps/rlark/pkg/agent/controllers/task/push_deployment.go index efc7833..a2d313e 100644 --- a/apps/rlark/pkg/agent/controllers/task/push_deployment.go +++ b/apps/rlark/pkg/agent/controllers/task/push_deployment.go @@ -66,8 +66,6 @@ func (r *pushDeploymentReconciler) Reconcile(ctx context.Context, req reconcile. func pushOwnsTask(task *rlarkv1alpha1.Task, agentType string, workload client.Object, kind string) bool { carrierType := taskapi.CarrierTypeKubernetesDeployment switch kind { - case raykubernetes.CarrierDaemonSet: - carrierType = taskapi.CarrierTypeKubernetesDaemonSet case raykubernetes.CarrierStatefulSet: carrierType = taskapi.CarrierTypeKubernetesStatefulSet } diff --git a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter.go b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter.go index 36170ff..1087d23 100644 --- a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter.go +++ b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter.go @@ -205,8 +205,6 @@ func (a *Adapter) Compile(_ context.Context, request workload.CompileRequest) (e switch kind { case CarrierDeployment: primary = BuildDeployment(task, namespace, spec, replicas) - case CarrierDaemonSet: - primary = BuildDaemonSet(task, namespace, spec) case CarrierStatefulSet: primary = BuildStatefulSet(task, namespace, spec, replicas) default: @@ -414,8 +412,6 @@ func kubernetesCarrierKind(carrier coordination.VersionedConfiguration) (string, switch carrier.Type { case taskapi.CarrierTypeKubernetesDeployment: return CarrierDeployment, nil - case taskapi.CarrierTypeKubernetesDaemonSet: - return CarrierDaemonSet, nil case taskapi.CarrierTypeKubernetesStatefulSet: return CarrierStatefulSet, nil default: @@ -436,11 +432,6 @@ func decodeKubernetesCarrier(carrier coordination.VersionedConfiguration) (strin return "", nil, fault.Wrap(fault.InvalidArgument, "decode replicated Kubernetes carrier parameters", err) } return kind, decoded.Replicas, nil - case CarrierDaemonSet: - if _, err := taskapi.DecodeParametersStrict[taskapi.KubernetesDaemonSetCarrierParametersV1](parameters); err != nil { - return "", nil, fault.Wrap(fault.InvalidArgument, "decode DaemonSet carrier parameters", err) - } - return kind, nil, nil default: return "", nil, fault.New(fault.InvalidArgument, "unsupported Kubernetes carrier type") } @@ -477,8 +468,6 @@ func gvkFor(object runtime.Object) schema.GroupVersionKind { switch object.(type) { case *appsv1.Deployment: return appsv1.SchemeGroupVersion.WithKind("Deployment") - case *appsv1.DaemonSet: - return appsv1.SchemeGroupVersion.WithKind("DaemonSet") case *appsv1.StatefulSet: return appsv1.SchemeGroupVersion.WithKind("StatefulSet") case *corev1.ConfigMap: @@ -526,12 +515,6 @@ func ObserveObject(kind string, manifest []byte, instances agentruntime.Instance return "", "", err } phase, message = DeploymentStatusPhase(&object) - case CarrierDaemonSet: - var object appsv1.DaemonSet - if err := json.Unmarshal(manifest, &object); err != nil { - return "", "", err - } - phase = DaemonSetStatusPhase(&object) case CarrierStatefulSet: var object appsv1.StatefulSet if err := json.Unmarshal(manifest, &object); err != nil { diff --git a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter_test.go b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter_test.go index 8a22592..69ca3a0 100644 --- a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter_test.go +++ b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/adapter_test.go @@ -31,9 +31,6 @@ func TestRayKubernetesAdaptersCompileLegacyWorkloadWithoutResourceDrift(t *testi {name: "deployment", kind: kubernetesworkload.CarrierDeployment, new: func() workload.Adapter { return kubernetesworkload.NewAdapter(workload.TypeID(taskapi.WorkloadTypeRay), namespace) }}, - {name: "daemonset", kind: kubernetesworkload.CarrierDaemonSet, new: func() workload.Adapter { - return kubernetesworkload.NewAdapter(workload.TypeID(taskapi.WorkloadTypeRay), namespace) - }}, {name: "statefulset", kind: kubernetesworkload.CarrierStatefulSet, new: func() workload.Adapter { return kubernetesworkload.NewAdapter(workload.TypeID(taskapi.WorkloadTypeRay), namespace) }}, @@ -83,8 +80,6 @@ func TestRayKubernetesAdaptersCompileLegacyWorkloadWithoutResourceDrift(t *testi switch test.kind { case kubernetesworkload.CarrierDeployment: expected = kubernetesworkload.BuildDeployment(task, "runtime", spec, carrierReplicas(task)) - case kubernetesworkload.CarrierDaemonSet: - expected = kubernetesworkload.BuildDaemonSet(task, "runtime", spec) case kubernetesworkload.CarrierStatefulSet: expected = kubernetesworkload.BuildStatefulSet(task, "runtime", spec, carrierReplicas(task)) } @@ -169,11 +164,6 @@ func TestRayKubernetesAdapterRejectsInvalidCarrierParameters(t *testing.T) { kind: kubernetesworkload.CarrierDeployment, parameters: `{"replicas":1,"unknown":true}`, }, - { - name: "daemonset replicas", - kind: kubernetesworkload.CarrierDaemonSet, - parameters: `{"replicas":1}`, - }, { name: "statefulset unknown field", kind: kubernetesworkload.CarrierStatefulSet, @@ -208,6 +198,26 @@ func TestRayKubernetesAdapterRejectsInvalidCarrierParameters(t *testing.T) { } } +func TestRayKubernetesAdapterRejectsDaemonSetCarrier(t *testing.T) { + task := compileTask(kubernetesworkload.CarrierDeployment) + task.Spec.Workload.Carrier.Type = "rlinf.io/kubernetes-daemonset" + task.Spec.Workload.Carrier.Parameters = apiextensionsv1.JSON{Raw: []byte(`{}`)} + snapshot, err := compatibility.DecodeTask(task) + if err != nil { + t.Fatal(err) + } + _, err = kubernetesworkload.NewAdapter( + workload.TypeID(taskapi.WorkloadTypeRay), + namespace, + ).Compile(context.Background(), workload.CompileRequest{ + Task: snapshot, + Runtime: agentruntime.Info{Kind: compatibility.RuntimeKindKubernetes}, + }) + if err == nil { + t.Fatal("DaemonSet carrier was accepted") + } +} + func TestRayKubernetesAdapterMarksRayResourcesOwned(t *testing.T) { task := compileTask(kubernetesworkload.CarrierDeployment) setRayRole(t, task, taskapi.RayRoleHead) @@ -444,8 +454,6 @@ func TestObserveStatusMatrixMatchesLegacyReducers(t *testing.T) { {name: "deployment unobserved", kind: kubernetesworkload.CarrierDeployment, object: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.DeploymentSpec{Replicas: &one}, Status: appsv1.DeploymentStatus{ObservedGeneration: 1}}, phase: rlarkv1alpha1.TaskPhasePending}, {name: "deployment deadline", kind: kubernetesworkload.CarrierDeployment, object: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.DeploymentSpec{Replicas: &one}, Status: appsv1.DeploymentStatus{ObservedGeneration: 2, Conditions: []appsv1.DeploymentCondition{{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionFalse, Reason: "ProgressDeadlineExceeded", Message: "deadline"}}}}, phase: rlarkv1alpha1.TaskPhaseFailed, message: "deadline"}, {name: "deployment replica failure", kind: kubernetesworkload.CarrierDeployment, object: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.DeploymentSpec{Replicas: &one}, Status: appsv1.DeploymentStatus{ObservedGeneration: 2, Conditions: []appsv1.DeploymentCondition{{Type: appsv1.DeploymentReplicaFailure, Status: corev1.ConditionTrue, Message: "replica"}}}}, phase: rlarkv1alpha1.TaskPhaseFailed, message: "replica"}, - {name: "daemonset partial", kind: kubernetesworkload.CarrierDaemonSet, object: &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Status: appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 2, UpdatedNumberScheduled: 1}}, phase: rlarkv1alpha1.TaskPhasePending}, - {name: "daemonset ready", kind: kubernetesworkload.CarrierDaemonSet, object: &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Status: appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 2, UpdatedNumberScheduled: 2, NumberReady: 2, NumberAvailable: 2}}, phase: rlarkv1alpha1.TaskPhaseRunning}, {name: "statefulset stopped", kind: kubernetesworkload.CarrierStatefulSet, object: &appsv1.StatefulSet{Spec: appsv1.StatefulSetSpec{Replicas: &zero}}, phase: rlarkv1alpha1.TaskPhaseStopped}, {name: "statefulset revision pending", kind: kubernetesworkload.CarrierStatefulSet, object: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.StatefulSetSpec{Replicas: &one}, Status: appsv1.StatefulSetStatus{ObservedGeneration: 2, UpdatedReplicas: 1, ReadyReplicas: 1, CurrentReplicas: 1, CurrentRevision: "old", UpdateRevision: "new"}}, phase: rlarkv1alpha1.TaskPhasePending}, {name: "statefulset ready", kind: kubernetesworkload.CarrierStatefulSet, object: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.StatefulSetSpec{Replicas: &one}, Status: appsv1.StatefulSetStatus{ObservedGeneration: 2, UpdatedReplicas: 1, ReadyReplicas: 1, CurrentReplicas: 1, CurrentRevision: "new", UpdateRevision: "new"}}, phase: rlarkv1alpha1.TaskPhaseRunning}, @@ -472,10 +480,6 @@ func compileTask(kind string) *rlarkv1alpha1.Task { replicas := int32(2) carrierParameters, _ := json.Marshal(taskapi.KubernetesReplicatedCarrierParametersV1{Replicas: &replicas}) carrierType := taskapi.CarrierTypeKubernetesDeployment - if kind == kubernetesworkload.CarrierDaemonSet { - carrierType = taskapi.CarrierTypeKubernetesDaemonSet - carrierParameters = []byte(`{}`) - } if kind == kubernetesworkload.CarrierStatefulSet { carrierType = taskapi.CarrierTypeKubernetesStatefulSet } diff --git a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/compatibility.go b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/compatibility.go index 9bf760f..6fed2a5 100644 --- a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/compatibility.go +++ b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/compatibility.go @@ -107,23 +107,6 @@ func BuildDeployment(task *rlarkv1alpha1.Task, namespace string, spec *rlarkv1al return result } -// BuildDaemonSet compiles a Kubernetes Task spec into a legacy-compatible DaemonSet. -func BuildDaemonSet(task *rlarkv1alpha1.Task, namespace string, spec *rlarkv1alpha1.TaskWorkloadSpec) *appsv1.DaemonSet { - result := &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: task.Name, - Namespace: namespace, - Annotations: WorkloadAnnotations(task), - }, - Spec: appsv1.DaemonSetSpec{ - Selector: WorkloadSelector(task), - Template: *spec.Template.DeepCopy(), - }, - } - tenant.Inherit(task, result) - return result -} - // BuildStatefulSet compiles a Kubernetes Task spec into a legacy-compatible StatefulSet. func BuildStatefulSet(task *rlarkv1alpha1.Task, namespace string, spec *rlarkv1alpha1.TaskWorkloadSpec, replicas *int32) *appsv1.StatefulSet { result := &appsv1.StatefulSet{ diff --git a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/observe.go b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/observe.go index 0071c53..4689879 100644 --- a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/observe.go +++ b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/observe.go @@ -33,14 +33,6 @@ func DeploymentStatusPhase(deployment *appsv1.Deployment) (rlarkv1alpha1.TaskPha return rlarkv1alpha1.TaskPhasePending, "" } -// DaemonSetStatusPhase projects a DaemonSet's current generation and replica counts. -func DaemonSetStatusPhase(daemonSet *appsv1.DaemonSet) rlarkv1alpha1.TaskPhase { - if daemonSet.Status.ObservedGeneration >= daemonSet.Generation && daemonSet.Status.DesiredNumberScheduled > 0 && daemonSet.Status.UpdatedNumberScheduled == daemonSet.Status.DesiredNumberScheduled && daemonSet.Status.NumberReady == daemonSet.Status.DesiredNumberScheduled && daemonSet.Status.NumberAvailable == daemonSet.Status.DesiredNumberScheduled { - return rlarkv1alpha1.TaskPhaseRunning - } - return rlarkv1alpha1.TaskPhasePending -} - // StatefulSetStatusPhase projects a StatefulSet's current revision and replica counts. func StatefulSetStatusPhase(statefulSet *appsv1.StatefulSet) rlarkv1alpha1.TaskPhase { desired := desiredReplicas(statefulSet.Spec.Replicas) diff --git a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations.go b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations.go index 6410510..798fdd1 100644 --- a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations.go +++ b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations.go @@ -11,11 +11,10 @@ import ( const ( CarrierDeployment = "Deployment" - CarrierDaemonSet = "DaemonSet" CarrierStatefulSet = "StatefulSet" ) -// Operations is the compatibility port used by the three current Kubernetes +// Operations is the compatibility port used by the current Kubernetes // workload adapters. Its implementation remains in the Task controller while // object operations are migrated incrementally into the Kubernetes runtime. type Operations interface { diff --git a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations_test.go b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations_test.go index 226182f..40f3e0f 100644 --- a/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations_test.go +++ b/apps/rlark/pkg/agent/controllers/task/workload/ray/kubernetes/operations_test.go @@ -37,9 +37,6 @@ func TestBuiltinAdaptersKeepTheirOwnWorkloadKind(t *testing.T) { {name: "deployment", kind: raykubernetes.CarrierDeployment, new: func(o *operations) adapter { return raykubernetes.NewLegacyAdapter(raykubernetes.CarrierDeployment, o) }}, - {name: "daemonset", kind: raykubernetes.CarrierDaemonSet, new: func(o *operations) adapter { - return raykubernetes.NewLegacyAdapter(raykubernetes.CarrierDaemonSet, o) - }}, {name: "statefulset", kind: raykubernetes.CarrierStatefulSet, new: func(o *operations) adapter { return raykubernetes.NewLegacyAdapter(raykubernetes.CarrierStatefulSet, o) }}, diff --git a/apps/rlark/pkg/controllermanager/cluster/cluster_cleanup_test.go b/apps/rlark/pkg/controllermanager/cluster/cluster_cleanup_test.go new file mode 100644 index 0000000..c6faaba --- /dev/null +++ b/apps/rlark/pkg/controllermanager/cluster/cluster_cleanup_test.go @@ -0,0 +1,35 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1" +) + +func TestDeleteClusterResourcesRejectsEmptyAgentNamespace(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add RLark scheme: %v", err) + } + client := fake.NewClientBuilder().WithScheme(scheme).Build() + reconciler := Reconciler{Client: client} + cluster := clusterWithSpec(rlarkv1alpha1.ClusterSpec{}) + + pending, err := reconciler.deleteClusterResources(context.Background(), &cluster) + if err == nil { + t.Fatal("expected empty agent namespace to be rejected") + } + if pending { + t.Fatal("cleanup should not be pending when agent namespace is invalid") + } + if !strings.Contains(err.Error(), "agent namespace is empty") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/apps/rlark/pkg/controllermanager/cluster/cluster_controller.go b/apps/rlark/pkg/controllermanager/cluster/cluster_controller.go index 44f1710..e2bde20 100644 --- a/apps/rlark/pkg/controllermanager/cluster/cluster_controller.go +++ b/apps/rlark/pkg/controllermanager/cluster/cluster_controller.go @@ -14,6 +14,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" controllerconfig "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -21,7 +22,12 @@ import ( ) const ( - heartbeatLeaseName = "heartbeat" + heartbeatLeaseName = "heartbeat" + CleanupFinalizer = "clusters.rlinf.io/cleanup" + cleanupRequeue = time.Second + drainInitialBackoff = time.Second + drainMaxBackoff = time.Minute + drainTimeout = 30 * time.Minute ) // Reconciler reconciles Cluster status from the agent heartbeat and registered Nodes. @@ -34,15 +40,84 @@ type Reconciler struct { // +kubebuilder:rbac:groups=rlinf.io,resources=clusters,verbs=get;list;watch // +kubebuilder:rbac:groups=rlinf.io,resources=clusters/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=rlinf.io,resources=nodes,verbs=get;list;watch -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch +// +kubebuilder:rbac:groups=rlinf.io,resources=clusters/finalizers,verbs=update +// +kubebuilder:rbac:groups=rlinf.io,resources=nodes,verbs=get;list;watch;delete +// +kubebuilder:rbac:groups=rlinf.io,resources=tasks,verbs=get;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;delete func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var cluster rlarkv1alpha1.Cluster if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + if !cluster.DeletionTimestamp.IsZero() { + return r.reconcileDeletion(ctx, &cluster) + } + if !controllerutil.ContainsFinalizer(&cluster, CleanupFinalizer) { + controllerutil.AddFinalizer(&cluster, CleanupFinalizer) + if err := r.Update(ctx, &cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("add Cluster cleanup finalizer: %w", err) + } + return ctrl.Result{}, nil + } + now := r.now() + conflict, err := r.agentNamespaceConflict(ctx, &cluster) + if err != nil { + return ctrl.Result{}, err + } + if conflict { + return r.reconcileAgentNamespaceConflict(ctx, &cluster, now) + } + beforeValidation := cluster.DeepCopy() + apimeta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: rlarkv1alpha1.ConditionTypeValidated, + Status: metav1.ConditionTrue, + Reason: "SpecValidated", + ObservedGeneration: cluster.Generation, + LastTransitionTime: metav1.NewTime(now), + }) + if !reflect.DeepEqual(cluster.Status.Conditions, beforeValidation.Status.Conditions) { + if err := r.Status().Patch(ctx, &cluster, client.MergeFrom(beforeValidation)); err != nil { + return ctrl.Result{}, fmt.Errorf("patch Cluster validation status: %w", err) + } + } + var drainRequeueAfter *time.Duration + if !cluster.Spec.Draining { + if err := r.clearDrainTaskMarkers(ctx, &cluster); err != nil { + return ctrl.Result{}, err + } + } + if cluster.Spec.Draining && !cluster.Spec.Delisted { + var complete bool + var pendingTasks []string + complete, pendingTasks, err = r.drainTasks(ctx, &cluster) + if err != nil { + return ctrl.Result{}, err + } + if complete { + before := cluster.DeepCopy() + clearDrainCondition(&cluster.Status.Conditions, now) + if !reflect.DeepEqual(cluster.Status.Conditions, before.Status.Conditions) { + if err := r.Status().Patch(ctx, &cluster, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("patch completed cluster drain status: %w", err) + } + } + } else { + before := cluster.DeepCopy() + startedAt := setDrainCondition(&cluster, now, pendingTasks) + delay := nextDrainRequeueAfter(now, startedAt) + drainRequeueAfter = &delay + if !reflect.DeepEqual(cluster.Status.Conditions, before.Status.Conditions) { + if err := r.Status().Patch(ctx, &cluster, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("patch cluster drain status: %w", err) + } + } + } + } + if cluster.Spec.AgentNamespace == "" { + return ctrl.Result{}, fmt.Errorf("cluster agent namespace is empty") + } phase, heartbeat, leaseDuration, totalNodes, onlineNodes, err := r.observe(ctx, &cluster, now) if err != nil { return ctrl.Result{}, err @@ -67,13 +142,130 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } } + if drainRequeueAfter != nil { + return ctrl.Result{RequeueAfter: *drainRequeueAfter}, nil + } return ctrl.Result{RequeueAfter: nextRequeueAfter(now, heartbeat, leaseDuration)}, nil } +func (r *Reconciler) agentNamespaceConflict(ctx context.Context, cluster *rlarkv1alpha1.Cluster) (bool, error) { + var clusters rlarkv1alpha1.ClusterList + if err := r.List(ctx, &clusters); err != nil { + return false, fmt.Errorf("list Clusters for agent namespace conflict: %w", err) + } + for i := range clusters.Items { + other := &clusters.Items[i] + if other.Name != cluster.Name && other.DeletionTimestamp.IsZero() && other.Spec.AgentNamespace == cluster.Spec.AgentNamespace { + return true, nil + } + } + return false, nil +} + +func (r *Reconciler) reconcileAgentNamespaceConflict(ctx context.Context, cluster *rlarkv1alpha1.Cluster, now time.Time) (ctrl.Result, error) { + before := cluster.DeepCopy() + cluster.Status.Phase = rlarkv1alpha1.ClusterOffline + apimeta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: rlarkv1alpha1.ConditionTypeValidated, + Status: metav1.ConditionFalse, + Reason: "AgentNamespaceConflict", + Message: "Another active Cluster claims this agent namespace.", + ObservedGeneration: cluster.Generation, + LastTransitionTime: metav1.NewTime(now), + }) + apimeta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: rlarkv1alpha1.ConditionTypeReady, + Status: metav1.ConditionFalse, + Reason: "AgentNamespaceConflict", + Message: "Cluster availability cannot be determined while its agent namespace is claimed by another Cluster.", + LastTransitionTime: metav1.NewTime(now), + }) + if !reflect.DeepEqual(cluster.Status, before.Status) { + if err := r.Status().Patch(ctx, cluster, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("patch Cluster namespace conflict status: %w", err) + } + } + return ctrl.Result{RequeueAfter: time.Minute}, nil +} + +func (r *Reconciler) reconcileDeletion(ctx context.Context, cluster *rlarkv1alpha1.Cluster) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(cluster, CleanupFinalizer) { + return ctrl.Result{}, nil + } + + pending, err := r.deleteClusterResources(ctx, cluster) + if err != nil { + return ctrl.Result{}, err + } + if pending { + return ctrl.Result{RequeueAfter: cleanupRequeue}, nil + } + + controllerutil.RemoveFinalizer(cluster, CleanupFinalizer) + if err := r.Update(ctx, cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("remove Cluster cleanup finalizer: %w", err) + } + return ctrl.Result{}, nil +} + +func (r *Reconciler) deleteClusterResources(ctx context.Context, cluster *rlarkv1alpha1.Cluster) (bool, error) { + if cluster.Spec.AgentNamespace == "" { + return false, fmt.Errorf("cluster agent namespace is empty") + } + + pending := false + + var tasks rlarkv1alpha1.TaskList + if err := r.List(ctx, &tasks, client.InNamespace(cluster.Spec.AgentNamespace)); err != nil { + return false, fmt.Errorf("list Tasks in agent namespace: %w", err) + } + for i := range tasks.Items { + task := &tasks.Items[i] + pending = true + if task.DeletionTimestamp.IsZero() { + if err := client.IgnoreNotFound(r.Delete(ctx, task)); err != nil { + return false, fmt.Errorf("delete Task %s/%s: %w", task.Namespace, task.Name, err) + } + } + } + + var nodes rlarkv1alpha1.NodeList + if err := r.List(ctx, &nodes, client.InNamespace(cluster.Spec.AgentNamespace)); err != nil { + return false, fmt.Errorf("list Nodes in agent namespace: %w", err) + } + for i := range nodes.Items { + node := &nodes.Items[i] + pending = true + if node.DeletionTimestamp.IsZero() { + if err := client.IgnoreNotFound(r.Delete(ctx, node)); err != nil { + return false, fmt.Errorf("delete Node %s/%s: %w", node.Namespace, node.Name, err) + } + } + } + + var heartbeat coordinationv1.Lease + if err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Spec.AgentNamespace, Name: heartbeatLeaseName}, &heartbeat); err != nil { + if !apierrors.IsNotFound(err) { + return false, fmt.Errorf("get agent heartbeat Lease: %w", err) + } + } else { + pending = true + if heartbeat.DeletionTimestamp.IsZero() { + if err := client.IgnoreNotFound(r.Delete(ctx, &heartbeat)); err != nil { + return false, fmt.Errorf("delete agent heartbeat Lease: %w", err) + } + } + } + + return pending, nil +} + func clusterReadyCondition(phase rlarkv1alpha1.ClusterPhase) (metav1.ConditionStatus, string) { switch phase { case rlarkv1alpha1.ClusterOnline: return metav1.ConditionTrue, "ClusterOnline" + case rlarkv1alpha1.ClusterDelisted: + return metav1.ConditionFalse, "ClusterDelisted" case rlarkv1alpha1.ClusterCordoned: return metav1.ConditionFalse, "ClusterCordoned" case rlarkv1alpha1.ClusterDraining: @@ -85,6 +277,95 @@ func clusterReadyCondition(phase rlarkv1alpha1.ClusterPhase) (metav1.ConditionSt } } +func (r *Reconciler) clearDrainTaskMarkers(ctx context.Context, cluster *rlarkv1alpha1.Cluster) error { + if cluster.Spec.AgentNamespace == "" { + return fmt.Errorf("cluster agent namespace is empty") + } + + var tasks rlarkv1alpha1.TaskList + if err := r.List(ctx, &tasks, client.InNamespace(cluster.Spec.AgentNamespace)); err != nil { + return fmt.Errorf("list Tasks in agent namespace: %w", err) + } + for i := range tasks.Items { + task := &tasks.Items[i] + stopped := task.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] == "true" + drainSuspended := task.Annotations[rlarkv1alpha1.ClusterDrainSuspendedAnnotation] == "true" + if !task.DeletionTimestamp.IsZero() || (!stopped && !drainSuspended) { + continue + } + before := task.DeepCopy() + delete(task.Annotations, rlarkv1alpha1.ClusterStoppedAnnotation) + if drainSuspended { + // Only resume Tasks whose suspension was written by the drain; a Task the + // user suspended before the drain keeps its own desired state. + delete(task.Annotations, rlarkv1alpha1.ClusterDrainSuspendedAnnotation) + task.Spec.Suspended = false + } + if err := r.Patch(ctx, task, client.MergeFrom(before)); err != nil { + return fmt.Errorf("clear cluster drain marker from Task %s/%s: %w", task.Namespace, task.Name, err) + } + } + return nil +} + +func taskTerminal(task *rlarkv1alpha1.Task) bool { + return task.Status.Phase == rlarkv1alpha1.TaskPhaseSucceeded || task.Status.Phase == rlarkv1alpha1.TaskPhaseFailed +} + +func taskHasJobOwner(task *rlarkv1alpha1.Task) bool { + for _, owner := range task.OwnerReferences { + if owner.APIVersion == rlarkv1alpha1.GroupVersion.String() && owner.Kind == "Job" { + return true + } + } + return false +} + +func (r *Reconciler) drainTasks(ctx context.Context, cluster *rlarkv1alpha1.Cluster) (bool, []string, error) { + if cluster.Spec.AgentNamespace == "" { + return false, nil, fmt.Errorf("cluster agent namespace is empty") + } + + var tasks rlarkv1alpha1.TaskList + if err := r.List(ctx, &tasks, client.InNamespace(cluster.Spec.AgentNamespace)); err != nil { + return false, nil, fmt.Errorf("list Tasks in agent namespace: %w", err) + } + + complete := true + pendingTasks := make([]string, 0, len(tasks.Items)) + for i := range tasks.Items { + task := &tasks.Items[i] + if !task.DeletionTimestamp.IsZero() || taskTerminal(task) { + continue + } + if task.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] == "true" { + if task.Status.ObservedGeneration >= task.Generation && task.Status.Phase == rlarkv1alpha1.TaskPhaseStopped { + continue + } + complete = false + pendingTasks = append(pendingTasks, task.Namespace+"/"+task.Name) + continue + } + complete = false + pendingTasks = append(pendingTasks, task.Namespace+"/"+task.Name) + before := task.DeepCopy() + if task.Annotations == nil { + task.Annotations = map[string]string{} + } + task.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] = "true" + if !taskHasJobOwner(task) && !task.Spec.Suspended { + // Standalone Tasks have no Job controller to honour the stopped marker, so + // the drain owns their suspension and records that it did. + task.Annotations[rlarkv1alpha1.ClusterDrainSuspendedAnnotation] = "true" + task.Spec.Suspended = true + } + if err := r.Patch(ctx, task, client.MergeFrom(before)); err != nil { + return false, nil, fmt.Errorf("mark Task %s/%s for cluster drain: %w", task.Namespace, task.Name, err) + } + } + return complete, pendingTasks, nil +} + func (r *Reconciler) observe(ctx context.Context, cluster *rlarkv1alpha1.Cluster, now time.Time) (rlarkv1alpha1.ClusterPhase, *metav1.Time, time.Duration, int32, int32, error) { var nodes rlarkv1alpha1.NodeList if err := r.List(ctx, &nodes, client.InNamespace(cluster.Spec.AgentNamespace)); err != nil { @@ -103,6 +384,9 @@ func (r *Reconciler) observe(ctx context.Context, cluster *rlarkv1alpha1.Cluster if err != nil { return "", nil, 0, 0, 0, err } + if cluster.Spec.Delisted { + return rlarkv1alpha1.ClusterDelisted, heartbeat, leaseDuration, totalNodes, onlineNodes, nil + } if cluster.Spec.Draining { return rlarkv1alpha1.ClusterDraining, heartbeat, leaseDuration, totalNodes, onlineNodes, nil } @@ -151,6 +435,64 @@ func sameTime(left, right *metav1.Time) bool { return left.Equal(right) } +func setDrainCondition(cluster *rlarkv1alpha1.Cluster, now time.Time, pendingTasks []string) *metav1.Time { + condition := apimeta.FindStatusCondition(cluster.Status.Conditions, rlarkv1alpha1.ConditionTypeProgressing) + startedAt := now + if condition != nil && condition.Status == metav1.ConditionTrue && condition.ObservedGeneration == cluster.Generation && (condition.Reason == "DrainInProgress" || condition.Reason == "DrainTimedOut") { + startedAt = condition.LastTransitionTime.Time + } + + reason := "DrainInProgress" + message := fmt.Sprintf("Waiting for %d Task(s) to report Stopped for their current generation.", len(pendingTasks)) + if now.Sub(startedAt) >= drainTimeout { + reason = "DrainTimedOut" + message = fmt.Sprintf("Drain exceeded the %s deadline; waiting for Task(s): %v.", drainTimeout, pendingTasks) + } + apimeta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: rlarkv1alpha1.ConditionTypeProgressing, + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + ObservedGeneration: cluster.Generation, + LastTransitionTime: metav1.NewTime(now), + }) + return &metav1.Time{Time: startedAt} +} + +func clearDrainCondition(conditions *[]metav1.Condition, now time.Time) { + condition := apimeta.FindStatusCondition(*conditions, rlarkv1alpha1.ConditionTypeProgressing) + if condition == nil || condition.Status != metav1.ConditionTrue || (condition.Reason != "DrainInProgress" && condition.Reason != "DrainTimedOut") { + return + } + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: rlarkv1alpha1.ConditionTypeProgressing, + Status: metav1.ConditionFalse, + Reason: "DrainComplete", + Message: "All Tasks reported Stopped for their current generation.", + ObservedGeneration: condition.ObservedGeneration, + LastTransitionTime: metav1.NewTime(now), + }) +} + +func nextDrainRequeueAfter(now time.Time, startedAt *metav1.Time) time.Duration { + if startedAt == nil { + return drainInitialBackoff + } + elapsed := now.Sub(startedAt.Time) + if elapsed >= drainTimeout { + return drainMaxBackoff + } + + delay := drainInitialBackoff + for elapsed >= delay && delay < drainMaxBackoff { + delay *= 2 + } + if remaining := drainTimeout - elapsed; remaining < delay { + return remaining + } + return delay +} + func nextRequeueAfter(now time.Time, heartbeat *metav1.Time, leaseDuration time.Duration) time.Duration { if heartbeat == nil || leaseDuration <= 0 { return time.Minute @@ -167,6 +509,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&rlarkv1alpha1.Cluster{}). Watches(&rlarkv1alpha1.Node{}, handler.EnqueueRequestsFromMapFunc(r.clusterForNode)). + Watches(&rlarkv1alpha1.Task{}, handler.EnqueueRequestsFromMapFunc(r.clusterForTask)). Watches(&coordinationv1.Lease{}, handler.EnqueueRequestsFromMapFunc(r.clustersForHeartbeatLease)). Named("cluster"). WithOptions(controllerconfig.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). @@ -180,13 +523,26 @@ func (r *Reconciler) clusterForNode(ctx context.Context, object client.Object) [ } requests := make([]reconcile.Request, 0, 1) for i := range clusters.Items { - if clusters.Items[i].Spec.AgentNamespace == object.GetNamespace() { + if clusters.Items[i].Spec.AgentNamespace == object.GetNamespace() && clusters.Items[i].DeletionTimestamp.IsZero() { requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKey{Name: clusters.Items[i].Name}}) } } return requests } +func (r *Reconciler) clusterForTask(ctx context.Context, object client.Object) []reconcile.Request { + var clusters rlarkv1alpha1.ClusterList + if err := r.List(ctx, &clusters); err != nil { + return nil + } + for i := range clusters.Items { + if clusters.Items[i].Spec.AgentNamespace == object.GetNamespace() && clusters.Items[i].Spec.Draining { + return []reconcile.Request{{NamespacedName: client.ObjectKey{Name: clusters.Items[i].Name}}} + } + } + return nil +} + func (r *Reconciler) clustersForHeartbeatLease(ctx context.Context, object client.Object) []reconcile.Request { if object.GetName() != heartbeatLeaseName { return nil @@ -199,7 +555,7 @@ func (r *Reconciler) clustersForHeartbeatLease(ctx context.Context, object clien requests := make([]reconcile.Request, 0, 1) for _, cluster := range clusters.Items { - if cluster.Spec.AgentNamespace == object.GetNamespace() { + if cluster.Spec.AgentNamespace == object.GetNamespace() && cluster.DeletionTimestamp.IsZero() { requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKey{Name: cluster.Name}}) } } diff --git a/apps/rlark/pkg/controllermanager/cluster/cluster_controller_test.go b/apps/rlark/pkg/controllermanager/cluster/cluster_controller_test.go index 01e92d1..3cc5f78 100644 --- a/apps/rlark/pkg/controllermanager/cluster/cluster_controller_test.go +++ b/apps/rlark/pkg/controllermanager/cluster/cluster_controller_test.go @@ -6,6 +6,7 @@ import ( "time" coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -63,6 +64,12 @@ func TestObserve(t *testing.T) { lease: testLease(freshHeartbeat, &leaseDuration), wantPhase: rlarkv1alpha1.ClusterDowngraded, }, + { + name: "delisted overrides observed availability", + cluster: clusterWithSpec(rlarkv1alpha1.ClusterSpec{AgentNamespace: "rlark-agent-cluster-a", Delisted: true}), + lease: testLease(staleHeartbeat, &leaseDuration), + wantPhase: rlarkv1alpha1.ClusterDelisted, + }, { name: "cordoned overrides observed availability", cluster: clusterWithSpec(rlarkv1alpha1.ClusterSpec{AgentNamespace: "rlark-agent-cluster-a", Cordoned: true}), @@ -148,6 +155,298 @@ func testLeaseWithoutRenewal(leaseDuration *int32) *coordinationv1.Lease { } } +func TestReconcileReportsAgentNamespaceConflict(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + clusterA := clusterWithSpec(rlarkv1alpha1.ClusterSpec{AgentNamespace: "shared-agent"}) + clusterB := clusterWithSpec(rlarkv1alpha1.ClusterSpec{AgentNamespace: "shared-agent"}) + clusterB.Name = "cluster-b" + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add RLark scheme: %v", err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&rlarkv1alpha1.Cluster{}).WithRuntimeObjects(&clusterA, &clusterB).Build() + reconciler := Reconciler{Client: client, Now: func() time.Time { return now }} + + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(clusterA.Name)); err != nil { + t.Fatalf("initialize Cluster finalizer: %v", err) + } + result, err := reconciler.Reconcile(context.Background(), reconcileRequest(clusterA.Name)) + if err != nil { + t.Fatalf("reconcile conflict: %v", err) + } + if result.RequeueAfter != time.Minute { + t.Fatalf("requeue after = %s, want %s", result.RequeueAfter, time.Minute) + } + + var updated rlarkv1alpha1.Cluster + if err := client.Get(context.Background(), types.NamespacedName{Name: clusterA.Name}, &updated); err != nil { + t.Fatalf("get Cluster: %v", err) + } + if updated.Status.Phase != rlarkv1alpha1.ClusterOffline { + t.Fatalf("phase = %q, want %q", updated.Status.Phase, rlarkv1alpha1.ClusterOffline) + } + validated := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeValidated) + if validated == nil || validated.Status != metav1.ConditionFalse || validated.Reason != "AgentNamespaceConflict" || validated.ObservedGeneration != updated.Generation { + t.Fatalf("unexpected Validated condition: %#v", validated) + } + ready := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "AgentNamespaceConflict" { + t.Fatalf("unexpected Ready condition: %#v", ready) + } +} + +func TestReconcileMarksClusterValidatedWithoutAgentNamespaceConflict(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + cluster := testCluster() + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add RLark scheme: %v", err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatalf("add coordination scheme: %v", err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&rlarkv1alpha1.Cluster{}).WithRuntimeObjects(&cluster).Build() + reconciler := Reconciler{Client: client, Now: func() time.Time { return now }} + + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatalf("initialize Cluster finalizer: %v", err) + } + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatalf("reconcile Cluster: %v", err) + } + + var updated rlarkv1alpha1.Cluster + if err := client.Get(context.Background(), types.NamespacedName{Name: cluster.Name}, &updated); err != nil { + t.Fatalf("get Cluster: %v", err) + } + validated := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeValidated) + if validated == nil || validated.Status != metav1.ConditionTrue || validated.Reason != "SpecValidated" || validated.ObservedGeneration != updated.Generation { + t.Fatalf("unexpected Validated condition: %#v", validated) + } +} + +func TestReconcileRecoversConditionsAfterAgentNamespaceConflictClears(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + leaseDuration := int32(30) + freshHeartbeat := metav1.NewMicroTime(now.Add(-5 * time.Second)) + clusterA := clusterWithSpec(rlarkv1alpha1.ClusterSpec{AgentNamespace: "shared-agent"}) + clusterB := clusterWithSpec(rlarkv1alpha1.ClusterSpec{AgentNamespace: "shared-agent"}) + clusterB.Name = "cluster-b" + node := testNode("node-1", rlarkv1alpha1.NodeOnline) + node.Namespace = clusterA.Spec.AgentNamespace + lease := testLease(freshHeartbeat, &leaseDuration) + lease.Namespace = clusterA.Spec.AgentNamespace + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add RLark scheme: %v", err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatalf("add coordination scheme: %v", err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&rlarkv1alpha1.Cluster{}).WithRuntimeObjects(&clusterA, &clusterB, &node, lease).Build() + reconciler := Reconciler{Client: client, Now: func() time.Time { return now }} + + for range 2 { + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(clusterA.Name)); err != nil { + t.Fatalf("reconcile conflicting Cluster: %v", err) + } + } + + if err := client.Delete(context.Background(), &clusterB); err != nil { + t.Fatalf("delete conflicting Cluster: %v", err) + } + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(clusterA.Name)); err != nil { + t.Fatalf("reconcile recovered Cluster: %v", err) + } + + var updated rlarkv1alpha1.Cluster + if err := client.Get(context.Background(), types.NamespacedName{Name: clusterA.Name}, &updated); err != nil { + t.Fatalf("get Cluster: %v", err) + } + validated := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeValidated) + if validated == nil || validated.Status != metav1.ConditionTrue || validated.Reason != "SpecValidated" || validated.ObservedGeneration != updated.Generation { + t.Fatalf("unexpected Validated condition: %#v", validated) + } + ready := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != "ClusterOnline" { + t.Fatalf("unexpected Ready condition: %#v", ready) + } + if updated.Status.Phase != rlarkv1alpha1.ClusterOnline { + t.Fatalf("phase = %q, want %q", updated.Status.Phase, rlarkv1alpha1.ClusterOnline) + } +} + +func TestReconcileFinishesDrainAndRestoresObservedPhase(t *testing.T) { + t.Parallel() + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + leaseDuration := int32(30) + staleHeartbeat := metav1.NewMicroTime(now.Add(-time.Minute)) + cluster := clusterWithSpec(rlarkv1alpha1.ClusterSpec{ + AgentNamespace: "rlark-agent-cluster-a", + Cordoned: true, + Draining: true, + DrainPreviousCordoned: false, + }) + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add RLark scheme: %v", err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatalf("add coordination scheme: %v", err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&rlarkv1alpha1.Cluster{}).WithRuntimeObjects(&cluster, testLease(staleHeartbeat, &leaseDuration)).Build() + reconciler := Reconciler{Client: client, Now: func() time.Time { return now }} + + for range 2 { + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatalf("reconcile completed drain: %v", err) + } + } + + var updated rlarkv1alpha1.Cluster + if err := client.Get(context.Background(), types.NamespacedName{Name: cluster.Name}, &updated); err != nil { + t.Fatalf("get updated cluster: %v", err) + } + if !updated.Spec.Draining || !updated.Spec.Cordoned || updated.Spec.DrainPreviousCordoned { + t.Fatalf("expected drain request to remain in Spec, got %#v", updated.Spec) + } + if updated.Status.Phase != rlarkv1alpha1.ClusterDraining { + t.Fatalf("expected phase %q while drain remains requested, got %q", rlarkv1alpha1.ClusterDraining, updated.Status.Phase) + } + condition := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeReady) + if condition == nil || condition.Reason != "ClusterDraining" { + t.Fatalf("unexpected Ready condition: %#v", condition) + } +} + +func TestReconcileReportsDrainTimeout(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + cluster := clusterWithSpec(rlarkv1alpha1.ClusterSpec{AgentNamespace: "rlark-agent-cluster-a", Draining: true}) + task := drainTestTask("task-1", cluster.Spec.AgentNamespace) + task.Annotations = map[string]string{rlarkv1alpha1.ClusterStoppedAnnotation: "true"} + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&rlarkv1alpha1.Cluster{}).WithRuntimeObjects(&cluster, task).Build() + startedAt := now.Add(-drainTimeout) + reconciler := Reconciler{Client: client, Now: func() time.Time { return startedAt }} + + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatalf("initialize Cluster finalizer: %v", err) + } + result, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)) + if err != nil { + t.Fatalf("start drain: %v", err) + } + if result.RequeueAfter != drainInitialBackoff { + t.Fatalf("requeue after = %s, want %s", result.RequeueAfter, drainInitialBackoff) + } + + reconciler.Now = func() time.Time { return now } + result, err = reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)) + if err != nil { + t.Fatalf("report drain timeout: %v", err) + } + if result.RequeueAfter != drainMaxBackoff { + t.Fatalf("requeue after = %s, want %s", result.RequeueAfter, drainMaxBackoff) + } + + var updated rlarkv1alpha1.Cluster + if err := client.Get(context.Background(), types.NamespacedName{Name: cluster.Name}, &updated); err != nil { + t.Fatalf("get Cluster: %v", err) + } + condition := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeProgressing) + if condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != "DrainTimedOut" || condition.ObservedGeneration != updated.Generation { + t.Fatalf("unexpected Progressing condition: %#v", condition) + } +} + +func TestReconcileResumesTimedOutDrainAfterRestart(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + cluster := clusterWithSpec(rlarkv1alpha1.ClusterSpec{ + AgentNamespace: "rlark-agent-cluster-a", + Cordoned: true, + Draining: true, + DrainPreviousCordoned: false, + }) + task := drainTestTask("task-1", cluster.Spec.AgentNamespace) + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&rlarkv1alpha1.Cluster{}).WithRuntimeObjects(&cluster, task).Build() + first := Reconciler{Client: client, Now: func() time.Time { return now }} + + for range 2 { + if _, err := first.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatalf("start drain: %v", err) + } + } + + restarted := Reconciler{Client: client, Now: func() time.Time { return now.Add(drainTimeout) }} + result, err := restarted.Reconcile(context.Background(), reconcileRequest(cluster.Name)) + if err != nil { + t.Fatalf("resume drain after restart: %v", err) + } + if result.RequeueAfter != drainMaxBackoff { + t.Fatalf("requeue after = %s, want %s", result.RequeueAfter, drainMaxBackoff) + } + + var updatedTask rlarkv1alpha1.Task + if err := client.Get(context.Background(), types.NamespacedName{Namespace: task.Namespace, Name: task.Name}, &updatedTask); err != nil { + t.Fatalf("get Task: %v", err) + } + updatedTask.Status.Phase = rlarkv1alpha1.TaskPhaseStopped + updatedTask.Status.ObservedGeneration = updatedTask.Generation + if err := client.Update(context.Background(), &updatedTask); err != nil { + t.Fatalf("update stopped Task: %v", err) + } + + if _, err := restarted.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatalf("finish drain after restart: %v", err) + } + var updated rlarkv1alpha1.Cluster + if err := client.Get(context.Background(), types.NamespacedName{Name: cluster.Name}, &updated); err != nil { + t.Fatalf("get Cluster: %v", err) + } + if !updated.Spec.Draining || !updated.Spec.Cordoned || updated.Spec.DrainPreviousCordoned { + t.Fatalf("expected drain request to remain in Spec, got %#v", updated.Spec) + } + condition := apimeta.FindStatusCondition(updated.Status.Conditions, rlarkv1alpha1.ConditionTypeProgressing) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != "DrainComplete" { + t.Fatalf("unexpected Progressing condition: %#v", condition) + } +} + +func TestNextDrainRequeueAfter(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + startedAt := metav1.NewTime(now) + if got := nextDrainRequeueAfter(now, &startedAt); got != drainInitialBackoff { + t.Fatalf("initial drain requeue = %s, want %s", got, drainInitialBackoff) + } + startedAt = metav1.NewTime(now.Add(-2 * time.Second)) + if got := nextDrainRequeueAfter(now, &startedAt); got != 4*time.Second { + t.Fatalf("exponential drain requeue = %s, want %s", got, 4*time.Second) + } + startedAt = metav1.NewTime(now.Add(-drainTimeout)) + if got := nextDrainRequeueAfter(now, &startedAt); got != drainMaxBackoff { + t.Fatalf("timed out drain requeue = %s, want %s", got, drainMaxBackoff) + } +} + func TestReconcileUpdatesStatus(t *testing.T) { now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) freshHeartbeat := metav1.NewMicroTime(now.Add(-5 * time.Second)) @@ -165,8 +464,10 @@ func TestReconcileUpdatesStatus(t *testing.T) { client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&rlarkv1alpha1.Cluster{}).WithRuntimeObjects(&cluster, &node, testLease(freshHeartbeat, &leaseDuration)).Build() reconciler := Reconciler{Client: client, Now: func() time.Time { return now }} - if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { - t.Fatalf("reconcile cluster: %v", err) + for range 2 { + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatalf("reconcile cluster: %v", err) + } } var updated rlarkv1alpha1.Cluster @@ -185,7 +486,8 @@ func TestReconcileUpdatesStatus(t *testing.T) { if err != nil { t.Fatalf("reconcile unchanged cluster: %v", err) } - if result.RequeueAfter != 25*time.Second { + const expectedRequeueAfter = 25 * time.Second + if result.RequeueAfter != expectedRequeueAfter { t.Fatalf("expected requeue after 25s, got %s", result.RequeueAfter) } var unchanged rlarkv1alpha1.Cluster @@ -197,6 +499,101 @@ func TestReconcileUpdatesStatus(t *testing.T) { } } +func TestReconcileDeletionRemovesFinalizerAfterResourcesAreGone(t *testing.T) { + deletedAt := metav1.Now() + cluster := &rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-a", + DeletionTimestamp: &deletedAt, + Finalizers: []string{CleanupFinalizer}, + }, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "rlark-agent-cluster-a"}, + } + node := testNode("node-1", rlarkv1alpha1.NodeOnline) + node.Namespace = cluster.Spec.AgentNamespace + task := drainTestTask("task-1", "rlark-agent-cluster-a") + leaseDuration := int32(30) + lease := testLease(metav1.NewMicroTime(time.Now()), &leaseDuration) + lease.Namespace = cluster.Spec.AgentNamespace + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, &node, task, lease).Build() + reconciler := Reconciler{Client: client} + + result, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != cleanupRequeue { + t.Fatalf("requeue after = %s, want %s", result.RequeueAfter, cleanupRequeue) + } + + for range 2 { + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatal(err) + } + } + var updated rlarkv1alpha1.Cluster + if err := client.Get(context.Background(), types.NamespacedName{Name: cluster.Name}, &updated); !apierrors.IsNotFound(err) { + t.Fatalf("Cluster should be deleted after cleanup, got: %v", err) + } +} + +func TestReconcileDeletionRemovesResourcesWithStaleClusterLabels(t *testing.T) { + deletedAt := metav1.Now() + cluster := &rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-a", + DeletionTimestamp: &deletedAt, + Finalizers: []string{CleanupFinalizer}, + }, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "shared-agent"}, + } + node := testNode("node-b", rlarkv1alpha1.NodeOnline) + node.Namespace = cluster.Spec.AgentNamespace + node.Labels = map[string]string{rlarkv1alpha1.LabelClusterID: "cluster-b"} + task := drainTestTask("task-b", cluster.Spec.AgentNamespace) + task.Labels = map[string]string{rlarkv1alpha1.LabelClusterID: "cluster-b"} + leaseDuration := int32(30) + lease := testLease(metav1.NewMicroTime(time.Now()), &leaseDuration) + lease.Namespace = cluster.Spec.AgentNamespace + lease.Labels = map[string]string{rlarkv1alpha1.LabelClusterID: "cluster-b"} + + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, &node, task, lease).Build() + reconciler := Reconciler{Client: client} + + for range 3 { + if _, err := reconciler.Reconcile(context.Background(), reconcileRequest(cluster.Name)); err != nil { + t.Fatal(err) + } + } + if err := client.Get(context.Background(), types.NamespacedName{Namespace: node.Namespace, Name: node.Name}, &rlarkv1alpha1.Node{}); !apierrors.IsNotFound(err) { + t.Fatalf("Node with stale label should be deleted, got: %v", err) + } + if err := client.Get(context.Background(), types.NamespacedName{Namespace: task.Namespace, Name: task.Name}, &rlarkv1alpha1.Task{}); !apierrors.IsNotFound(err) { + t.Fatalf("Task with stale label should be deleted, got: %v", err) + } + if err := client.Get(context.Background(), types.NamespacedName{Namespace: lease.Namespace, Name: lease.Name}, &coordinationv1.Lease{}); !apierrors.IsNotFound(err) { + t.Fatalf("Lease with stale label should be deleted, got: %v", err) + } + if err := client.Get(context.Background(), types.NamespacedName{Name: cluster.Name}, &rlarkv1alpha1.Cluster{}); !apierrors.IsNotFound(err) { + t.Fatalf("Cluster should be deleted after namespace cleanup, got: %v", err) + } +} + func TestNextRequeueAfter(t *testing.T) { now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) heartbeat := metav1.NewTime(now.Add(-5 * time.Second)) diff --git a/apps/rlark/pkg/controllermanager/cluster/cluster_drain_test.go b/apps/rlark/pkg/controllermanager/cluster/cluster_drain_test.go new file mode 100644 index 0000000..f0780e6 --- /dev/null +++ b/apps/rlark/pkg/controllermanager/cluster/cluster_drain_test.go @@ -0,0 +1,248 @@ +package cluster + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1" +) + +func TestDrainTasksStopsOnlyTasksInAgentNamespace(t *testing.T) { + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "cluster-a", Draining: true}, + } + targetTask := drainTestTask("target", "cluster-a") + targetTask.OwnerReferences = []metav1.OwnerReference{{APIVersion: rlarkv1alpha1.GroupVersion.String(), Kind: "Job", Name: "job-a"}} + otherTask := drainTestTask("other", "cluster-b") + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, targetTask, otherTask).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + complete, _, err := r.drainTasks(context.Background(), cluster) + if err != nil { + t.Fatal(err) + } + if complete { + t.Fatal("first drain pass must wait for stopped task observation") + } + + var got rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKeyFromObject(targetTask), &got); err != nil { + t.Fatal(err) + } + if got.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "true" || got.Spec.Suspended { + t.Fatalf("cluster drain must only mark target task for the Job controller: %#v", got) + } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(otherTask), &got); err != nil { + t.Fatal(err) + } + if got.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "" || got.Spec.Suspended { + t.Fatalf("task outside cluster was changed: %#v", got) + } + + complete, _, err = r.drainTasks(context.Background(), cluster) + if err != nil { + t.Fatal(err) + } + if complete { + t.Fatal("drain must wait for the agent to observe the stopped task") + } + + if err := c.Get(context.Background(), client.ObjectKeyFromObject(targetTask), &got); err != nil { + t.Fatal(err) + } + got.Status.Phase = rlarkv1alpha1.TaskPhaseStopped + got.Status.ObservedGeneration = got.Generation + if err := c.Update(context.Background(), &got); err != nil { + t.Fatal(err) + } + + complete, _, err = r.drainTasks(context.Background(), cluster) + if err != nil { + t.Fatal(err) + } + if !complete { + t.Fatal("drain should complete after the agent observes every task stopped") + } +} + +func TestDrainTasksSkipsTerminalTasksAndSuspendsStandaloneTasks(t *testing.T) { + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "cluster-a", Draining: true}, + } + terminal := drainTestTask("terminal", "cluster-a") + terminal.Status.Phase = rlarkv1alpha1.TaskPhaseSucceeded + standalone := drainTestTask("standalone", "cluster-a") + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, terminal, standalone).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + complete, pending, err := r.drainTasks(context.Background(), cluster) + if err != nil { + t.Fatal(err) + } + if complete || len(pending) != 1 || pending[0] != "cluster-a/standalone" { + t.Fatalf("unexpected drain result: complete=%t pending=%v", complete, pending) + } + + var gotTerminal, gotStandalone rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKeyFromObject(terminal), &gotTerminal); err != nil { + t.Fatal(err) + } + if gotTerminal.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "" || gotTerminal.Spec.Suspended { + t.Fatalf("terminal task was changed: %#v", gotTerminal) + } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(standalone), &gotStandalone); err != nil { + t.Fatal(err) + } + if gotStandalone.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "true" || !gotStandalone.Spec.Suspended { + t.Fatalf("standalone task was not suspended for drain: %#v", gotStandalone) + } + + gotStandalone.Status.Phase = rlarkv1alpha1.TaskPhaseStopped + gotStandalone.Status.ObservedGeneration = gotStandalone.Generation + if err := c.Update(context.Background(), &gotStandalone); err != nil { + t.Fatal(err) + } + complete, pending, err = r.drainTasks(context.Background(), cluster) + if err != nil { + t.Fatal(err) + } + if !complete || len(pending) != 0 { + t.Fatalf("drain result after standalone stop: complete=%t pending=%v", complete, pending) + } +} + +func TestClearDrainTaskMarkersOnlyClearsTasksInAgentNamespace(t *testing.T) { + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "cluster-a"}, + } + targetTask := drainTestTask("target", "cluster-a") + targetTask.Annotations = map[string]string{rlarkv1alpha1.ClusterStoppedAnnotation: "true", "keep": "value"} + otherTask := drainTestTask("other", "cluster-b") + otherTask.Annotations = map[string]string{rlarkv1alpha1.ClusterStoppedAnnotation: "true"} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, targetTask, otherTask).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + if err := r.clearDrainTaskMarkers(context.Background(), cluster); err != nil { + t.Fatal(err) + } + + var got rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKeyFromObject(targetTask), &got); err != nil { + t.Fatal(err) + } + if got.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "" || got.Annotations["keep"] != "value" { + t.Fatalf("target Task drain marker was not cleared safely: %#v", got.Annotations) + } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(otherTask), &got); err != nil { + t.Fatal(err) + } + if got.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "true" { + t.Fatalf("Task outside cluster was changed: %#v", got.Annotations) + } +} + +func TestClearDrainTaskMarkersResumesOnlyDrainSuspendedStandaloneTasks(t *testing.T) { + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "cluster-a", Draining: true}, + } + drainSuspended := drainTestTask("drain-suspended", "cluster-a") + userSuspended := drainTestTask("user-suspended", "cluster-a") + userSuspended.Spec.Suspended = true + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, drainSuspended, userSuspended).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + if _, _, err := r.drainTasks(context.Background(), cluster); err != nil { + t.Fatal(err) + } + + var got rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKeyFromObject(drainSuspended), &got); err != nil { + t.Fatal(err) + } + if !got.Spec.Suspended || got.Annotations[rlarkv1alpha1.ClusterDrainSuspendedAnnotation] != "true" { + t.Fatalf("drain must record that it suspended the standalone Task: %#v", got) + } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(userSuspended), &got); err != nil { + t.Fatal(err) + } + if got.Annotations[rlarkv1alpha1.ClusterDrainSuspendedAnnotation] != "" { + t.Fatalf("drain must not claim a suspension it did not write: %#v", got.Annotations) + } + + cluster.Spec.Draining = false + if err := r.clearDrainTaskMarkers(context.Background(), cluster); err != nil { + t.Fatal(err) + } + + if err := c.Get(context.Background(), client.ObjectKeyFromObject(drainSuspended), &got); err != nil { + t.Fatal(err) + } + if got.Spec.Suspended || + got.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "" || + got.Annotations[rlarkv1alpha1.ClusterDrainSuspendedAnnotation] != "" { + t.Fatalf("cancelling the drain must resume the Task it suspended: %#v", got) + } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(userSuspended), &got); err != nil { + t.Fatal(err) + } + if !got.Spec.Suspended || got.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "" { + t.Fatalf("cancelling the drain must keep a user suspension: %#v", got) + } +} + +func TestDrainTasksRejectsEmptyAgentNamespace(t *testing.T) { + scheme := runtime.NewScheme() + if err := rlarkv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Draining: true}} + task := drainTestTask("task-in-default-namespace", "") + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, task).Build() + reconciler := &Reconciler{Client: fakeClient, Scheme: scheme} + + if _, _, err := reconciler.drainTasks(context.Background(), cluster); err == nil { + t.Fatal("expected drain to reject an empty agent namespace") + } + if err := reconciler.clearDrainTaskMarkers(context.Background(), cluster); err == nil { + t.Fatal("expected marker cleanup to reject an empty agent namespace") + } + + var unchanged rlarkv1alpha1.Task + if err := fakeClient.Get(context.Background(), client.ObjectKeyFromObject(task), &unchanged); err != nil { + t.Fatalf("get Task: %v", err) + } + if unchanged.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "" { + t.Fatalf("task was marked for drain despite empty agent namespace: %#v", unchanged) + } +} + +func drainTestTask(name, namespace string) *rlarkv1alpha1.Task { + return &rlarkv1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + } +} diff --git a/apps/rlark/pkg/controllermanager/job/build.go b/apps/rlark/pkg/controllermanager/job/build.go index 946e7c2..153f737 100644 --- a/apps/rlark/pkg/controllermanager/job/build.go +++ b/apps/rlark/pkg/controllermanager/job/build.go @@ -32,23 +32,50 @@ func (r *Reconciler) resolveTaskNamespace(ctx context.Context, t *rlarkv1alpha1. } selector := labels.SelectorFromSet(simpleSelector) - var nodeList rlarkv1alpha1.NodeList if err := r.List(ctx, &nodeList, &client.ListOptions{LabelSelector: selector}); err != nil { return "", fmt.Errorf("list Nodes matching selector for Task template %q: %w", t.Name, err) } - if len(nodeList.Items) == 0 { return "", fmt.Errorf("no Nodes match selector for Task template %q", t.Name) } - sort.Slice(nodeList.Items, func(i, j int) bool { - if nodeList.Items[i].Namespace == nodeList.Items[j].Namespace { - return nodeList.Items[i].Name < nodeList.Items[j].Name + var clusterList rlarkv1alpha1.ClusterList + if err := r.List(ctx, &clusterList); err != nil { + return "", fmt.Errorf("list Clusters for Task template %q: %w", t.Name, err) + } + clustersByNamespace := make(map[string][]rlarkv1alpha1.Cluster) + for i := range clusterList.Items { + cluster := clusterList.Items[i] + clustersByNamespace[cluster.Spec.AgentNamespace] = append(clustersByNamespace[cluster.Spec.AgentNamespace], cluster) + } + + eligible := nodeList.Items[:0] + for _, node := range nodeList.Items { + if node.Spec.Unschedulable || node.Status.Phase != rlarkv1alpha1.NodeOnline { + continue + } + clusters := clustersByNamespace[node.Namespace] + if len(clusters) != 1 { + continue + } + cluster := clusters[0] + if !cluster.DeletionTimestamp.IsZero() || cluster.Spec.Cordoned || cluster.Spec.Draining || cluster.Spec.Delisted || (cluster.Status.Phase != rlarkv1alpha1.ClusterOnline && cluster.Status.Phase != rlarkv1alpha1.ClusterDowngraded) { + continue + } + eligible = append(eligible, node) + } + if len(eligible) == 0 { + return "", fmt.Errorf("no schedulable Nodes in an Online or Downgraded Cluster match selector for Task template %q", t.Name) + } + + sort.Slice(eligible, func(i, j int) bool { + if eligible[i].Namespace == eligible[j].Namespace { + return eligible[i].Name < eligible[j].Name } - return nodeList.Items[i].Namespace < nodeList.Items[j].Namespace + return eligible[i].Namespace < eligible[j].Namespace }) - return nodeList.Items[0].Namespace, nil + return eligible[0].Namespace, nil } func buildTaskStatusMap(job *rlarkv1alpha1.Job) map[string]*rlarkv1alpha1.JobTaskStatus { @@ -117,7 +144,7 @@ func setNetworkEnvironment(spec *rlarkv1alpha1.TaskSpec, domain string) { } func setSSHEnvironment(spec *rlarkv1alpha1.TaskSpec, publicKey string) { - parameters, _ := json.Marshal(taskapi.SSHEnvironmentParametersV1{AuthorizedKeys: []string{publicKey}}) + parameters, _ := json.Marshal(taskapi.SSHEnvironmentParametersV1{AuthorizedKeys: []string{publicKey}, Port: 2222}) environment := rlarkv1alpha1.TaskEnvironment{Type: taskapi.EnvironmentTypeSSH, Version: taskapi.EnvironmentVersionSSHV1, Parameters: apiextensionsv1.JSON{Raw: parameters}} for i := range spec.Environments { if spec.Environments[i].Type == environment.Type && spec.Environments[i].Version == environment.Version { diff --git a/apps/rlark/pkg/controllermanager/job/job_controller_test.go b/apps/rlark/pkg/controllermanager/job/job_controller_test.go index d8b7ffd..4eb9bfd 100644 --- a/apps/rlark/pkg/controllermanager/job/job_controller_test.go +++ b/apps/rlark/pkg/controllermanager/job/job_controller_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -104,6 +106,78 @@ func TestInvalidTaskTemplatePreventsAllTaskDispatch(t *testing.T) { } } +func TestClusterDrainAndJobReconcileDoNotFlipTaskSuspended(t *testing.T) { + job := &rlarkv1alpha1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "job", UID: types.UID("job-uid")}, + Spec: rlarkv1alpha1.JobSpec{Tasks: []rlarkv1alpha1.JobTaskTemplate{validJobTaskTemplate("worker")}}, + } + task := buildTask(job, job.Spec.Tasks[0], "job-worker", "default") + task.Annotations = map[string]string{rlarkv1alpha1.ClusterStoppedAnnotation: "true"} + scheme := jobTestScheme(t) + c := jobTestClientBuilder(scheme).WithObjects(task).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + if _, err := r.reconcileTaskInNamespace(context.Background(), job, job.Spec.Tasks[0], task.Namespace, logr.Discard()); err != nil { + t.Fatal(err) + } + + var got rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKeyFromObject(task), &got); err != nil { + t.Fatal(err) + } + if !got.Spec.Suspended { + t.Fatalf("Job controller did not apply the cluster drain stop request: %#v", got.Spec) + } + generation := got.Generation + + if _, err := r.reconcileTaskInNamespace(context.Background(), job, job.Spec.Tasks[0], got.Namespace, logr.Discard()); err != nil { + t.Fatal(err) + } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(&got), &got); err != nil { + t.Fatal(err) + } + if !got.Spec.Suspended || got.Generation != generation { + t.Fatalf("Job reconciliation flipped or rewrote drained Task: generation %d -> %d, spec %#v", generation, got.Generation, got.Spec) + } +} + +func TestClusterUndrainAllowsJobResume(t *testing.T) { + job := &rlarkv1alpha1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "job", UID: types.UID("job-uid")}, + Spec: rlarkv1alpha1.JobSpec{Tasks: []rlarkv1alpha1.JobTaskTemplate{validJobTaskTemplate("worker")}}, + } + task := buildTask(job, job.Spec.Tasks[0], "job-worker", "default") + task.Annotations = map[string]string{rlarkv1alpha1.ClusterStoppedAnnotation: "true"} + scheme := jobTestScheme(t) + c := jobTestClientBuilder(scheme).WithObjects(task).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + if _, err := r.reconcileTaskInNamespace(context.Background(), job, job.Spec.Tasks[0], task.Namespace, logr.Discard()); err != nil { + t.Fatalf("apply drain stop request: %v", err) + } + var got rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKeyFromObject(task), &got); err != nil { + t.Fatalf("get suspended Task: %v", err) + } + if !got.Spec.Suspended { + t.Fatalf("drain did not suspend Task: %#v", got.Spec) + } + + delete(got.Annotations, rlarkv1alpha1.ClusterStoppedAnnotation) + if err := c.Update(context.Background(), &got); err != nil { + t.Fatalf("clear drain marker: %v", err) + } + if _, err := r.reconcileTaskInNamespace(context.Background(), job, job.Spec.Tasks[0], got.Namespace, logr.Discard()); err != nil { + t.Fatalf("resume Job after undrain: %v", err) + } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(&got), &got); err != nil { + t.Fatalf("get resumed Task: %v", err) + } + if got.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "" || got.Spec.Suspended { + t.Fatalf("undrained Task did not resume with Job: %#v", got) + } +} + func TestDeleteOwnedTasksWaitsForCleanupAndFiltersOwnerUID(t *testing.T) { job := &rlarkv1alpha1.Job{ObjectMeta: metav1.ObjectMeta{Name: "job", UID: types.UID("job-uid")}} owned := &rlarkv1alpha1.Task{ObjectMeta: metav1.ObjectMeta{ diff --git a/apps/rlark/pkg/controllermanager/job/lifecycle_test.go b/apps/rlark/pkg/controllermanager/job/lifecycle_test.go index 2070f8f..b39f427 100644 --- a/apps/rlark/pkg/controllermanager/job/lifecycle_test.go +++ b/apps/rlark/pkg/controllermanager/job/lifecycle_test.go @@ -517,11 +517,17 @@ func TestResolvedNamespaceIsDeterministicAndMigratesTask(t *testing.T) { controller := true task.OwnerReferences = []metav1.OwnerReference{{APIVersion: rlarkv1alpha1.GroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, Controller: &controller}} nodes := []client.Object{ - &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "z", Namespace: "workers-b", Labels: map[string]string{"zone": "new"}}}, - &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "a", Namespace: "workers-a", Labels: map[string]string{"zone": "new"}}}, + &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "z", Namespace: "workers-b", Labels: map[string]string{"zone": "new"}}, Status: rlarkv1alpha1.NodeStatus{Phase: rlarkv1alpha1.NodeOnline}}, + &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "a", Namespace: "workers-a", Labels: map[string]string{"zone": "new"}}, Status: rlarkv1alpha1.NodeStatus{Phase: rlarkv1alpha1.NodeOnline}}, + } + clusters := []client.Object{ + &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "b"}, Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "workers-b"}, Status: rlarkv1alpha1.ClusterStatus{Phase: rlarkv1alpha1.ClusterOnline}}, + &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "a"}, Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "workers-a"}, Status: rlarkv1alpha1.ClusterStatus{Phase: rlarkv1alpha1.ClusterOnline}}, } scheme := jobTestScheme(t) - c := jobTestClientBuilder(scheme).WithObjects(append(nodes, task)...).Build() + objects := append(nodes, task) + objects = append(objects, clusters...) + c := jobTestClientBuilder(scheme).WithObjects(objects...).Build() r := &Reconciler{Client: c, Scheme: scheme} namespace, err := r.resolveTaskNamespace(context.Background(), &job.Spec.Tasks[0]) if err != nil || namespace != "workers-a" { @@ -537,6 +543,84 @@ func TestResolvedNamespaceIsDeterministicAndMigratesTask(t *testing.T) { } } +func TestResolveTaskNamespaceRejectsIneligibleClusters(t *testing.T) { + for _, tt := range []struct { + name string + cluster rlarkv1alpha1.Cluster + }{ + {name: "cordoned", cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Cordoned: true}}}, + {name: "draining", cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Draining: true}}}, + {name: "delisted", cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Delisted: true}}}, + } { + t.Run(tt.name, func(t *testing.T) { + template := rlarkv1alpha1.JobTaskTemplate{Name: "worker", TaskSpec: rlarkv1alpha1.TaskSpec{Placement: rlarkv1alpha1.TaskPlacementSpec{NodeSelector: map[string]string{"zone": "new"}}}} + job := &rlarkv1alpha1.Job{ObjectMeta: metav1.ObjectMeta{Name: "job", UID: types.UID("job-uid")}, Spec: rlarkv1alpha1.JobSpec{Tasks: []rlarkv1alpha1.JobTaskTemplate{template}}} + tt.cluster.Name = "cluster" + tt.cluster.Spec.AgentNamespace = "workers" + tt.cluster.Status.Phase = rlarkv1alpha1.ClusterOnline + node := &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", Namespace: "workers", Labels: map[string]string{"zone": "new"}}, Status: rlarkv1alpha1.NodeStatus{Phase: rlarkv1alpha1.NodeOnline}} + scheme := jobTestScheme(t) + c := jobTestClientBuilder(scheme).WithObjects(node, &tt.cluster).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + if _, err := r.dispatchTasks(context.Background(), job, logr.Discard()); err == nil { + t.Fatal("expected dispatch rejection") + } + var tasks rlarkv1alpha1.TaskList + if err := c.List(context.Background(), &tasks); err != nil { + t.Fatal(err) + } + if len(tasks.Items) != 0 { + t.Fatalf("created Tasks: %#v", tasks.Items) + } + }) + } +} + +func TestResolveTaskNamespaceAllowsDowngradedCluster(t *testing.T) { + template := rlarkv1alpha1.JobTaskTemplate{Name: "worker", TaskSpec: rlarkv1alpha1.TaskSpec{Placement: rlarkv1alpha1.TaskPlacementSpec{NodeSelector: map[string]string{"zone": "new"}}}} + job := &rlarkv1alpha1.Job{ObjectMeta: metav1.ObjectMeta{Name: "job", UID: types.UID("job-uid")}, Spec: rlarkv1alpha1.JobSpec{Tasks: []rlarkv1alpha1.JobTaskTemplate{template}}, Status: rlarkv1alpha1.JobStatus{Tasks: []rlarkv1alpha1.JobTaskStatus{{Name: "worker"}}}} + cluster := &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "workers"}, Status: rlarkv1alpha1.ClusterStatus{Phase: rlarkv1alpha1.ClusterDowngraded}} + node := &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", Namespace: "workers", Labels: map[string]string{"zone": "new"}}, Status: rlarkv1alpha1.NodeStatus{Phase: rlarkv1alpha1.NodeOnline}} + scheme := jobTestScheme(t) + c := jobTestClientBuilder(scheme).WithObjects(node, cluster).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + if _, err := r.dispatchTasks(context.Background(), job, logr.Discard()); err != nil { + t.Fatal(err) + } + var task rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKey{Name: "job-worker", Namespace: "workers"}, &task); err != nil { + t.Fatal(err) + } +} + +func TestResolveTaskNamespaceRecoversWhenClusterBecomesOnline(t *testing.T) { + template := rlarkv1alpha1.JobTaskTemplate{Name: "worker", TaskSpec: rlarkv1alpha1.TaskSpec{Placement: rlarkv1alpha1.TaskPlacementSpec{NodeSelector: map[string]string{"zone": "new"}}}} + job := &rlarkv1alpha1.Job{ObjectMeta: metav1.ObjectMeta{Name: "job", UID: types.UID("job-uid")}, Spec: rlarkv1alpha1.JobSpec{Tasks: []rlarkv1alpha1.JobTaskTemplate{template}}, Status: rlarkv1alpha1.JobStatus{Tasks: []rlarkv1alpha1.JobTaskStatus{{Name: "worker"}}}} + cluster := &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "workers", Cordoned: true}, Status: rlarkv1alpha1.ClusterStatus{Phase: rlarkv1alpha1.ClusterCordoned}} + node := &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", Namespace: "workers", Labels: map[string]string{"zone": "new"}}, Status: rlarkv1alpha1.NodeStatus{Phase: rlarkv1alpha1.NodeOnline}} + scheme := jobTestScheme(t) + c := jobTestClientBuilder(scheme).WithObjects(node, cluster).Build() + r := &Reconciler{Client: c, Scheme: scheme} + + if _, err := r.dispatchTasks(context.Background(), job, logr.Discard()); err == nil { + t.Fatal("expected cordoned cluster rejection") + } + cluster.Spec.Cordoned = false + cluster.Status.Phase = rlarkv1alpha1.ClusterOnline + if err := c.Update(context.Background(), cluster); err != nil { + t.Fatal(err) + } + if _, err := r.dispatchTasks(context.Background(), job, logr.Discard()); err != nil { + t.Fatal(err) + } + var task rlarkv1alpha1.Task + if err := c.Get(context.Background(), client.ObjectKey{Name: "job-worker", Namespace: "workers"}, &task); err != nil { + t.Fatal(err) + } +} + func TestDispatchResolvesNamespaceBeforeMutating(t *testing.T) { replicas := int32(2) template := rlarkv1alpha1.JobTaskTemplate{Name: "worker", TaskSpec: rlarkv1alpha1.TaskSpec{ @@ -547,9 +631,10 @@ func TestDispatchResolvesNamespaceBeforeMutating(t *testing.T) { task := buildTask(job, template, "job-worker", "workers") controller := true task.OwnerReferences = []metav1.OwnerReference{{APIVersion: rlarkv1alpha1.GroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, Controller: &controller}} - node := &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", Namespace: "workers", Labels: map[string]string{"zone": "new"}}} + node := &rlarkv1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", Namespace: "workers", Labels: map[string]string{"zone": "new"}}, Status: rlarkv1alpha1.NodeStatus{Phase: rlarkv1alpha1.NodeOnline}} + cluster := &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "workers"}, Status: rlarkv1alpha1.ClusterStatus{Phase: rlarkv1alpha1.ClusterOnline}} scheme := jobTestScheme(t) - baseClient := jobTestClientBuilder(scheme).WithObjects(task, node).Build() + baseClient := jobTestClientBuilder(scheme).WithObjects(task, node, cluster).Build() c := &failSecondNodeListClient{Client: baseClient} r := &Reconciler{Client: c, Scheme: scheme} if _, err := r.dispatchTasks(context.Background(), job, logr.Discard()); err != nil { diff --git a/apps/rlark/pkg/controllermanager/job/sync.go b/apps/rlark/pkg/controllermanager/job/sync.go index b3139f7..893b62c 100644 --- a/apps/rlark/pkg/controllermanager/job/sync.go +++ b/apps/rlark/pkg/controllermanager/job/sync.go @@ -191,9 +191,12 @@ func (r *Reconciler) reconcileTaskInNamespace( } return nil, controller.ErrRequeueAfterChildCleanup } - resuming := task.Spec.Suspended && !job.Spec.Suspended + resuming := task.Spec.Suspended && !job.Spec.Suspended && task.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] != "true" if (!taskTerminal(&task) || resuming) && !taskEqual(&task, job, t) { desired := buildTask(job, t, taskName, taskNamespace) + if task.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] == "true" { + desired.Spec.Suspended = true + } task.Spec = desired.Spec task.Annotations = utils.MergeAnnotations(task.Annotations, desired.Annotations) if err := r.Update(ctx, &task); err != nil { @@ -325,6 +328,9 @@ func (r *Reconciler) pruneTasksInNamespaces(ctx context.Context, job *rlarkv1alp func taskEqual(existing *rlarkv1alpha1.Task, job *rlarkv1alpha1.Job, t rlarkv1alpha1.JobTaskTemplate) bool { desired := buildTask(job, t, existing.Name, existing.Namespace) + if existing.Annotations[rlarkv1alpha1.ClusterStoppedAnnotation] == "true" { + desired.Spec.Suspended = true + } return reflect.DeepEqual(existing.Spec, desired.Spec) && existing.Annotations[RestartedAtAnnotation] == desired.Annotations[RestartedAtAnnotation] } diff --git a/apps/rlark/pkg/controllermanager/job/sync_test.go b/apps/rlark/pkg/controllermanager/job/sync_test.go index 93db0b4..2bc0fe3 100644 --- a/apps/rlark/pkg/controllermanager/job/sync_test.go +++ b/apps/rlark/pkg/controllermanager/job/sync_test.go @@ -154,6 +154,23 @@ func TestPendingCanReachTerminalStates(t *testing.T) { } } +func TestBuildTaskUsesOnlyJobSuspension(t *testing.T) { + job := &rlarkv1alpha1.Job{} + task := buildTask(job, rlarkv1alpha1.JobTaskTemplate{ + Name: "worker", + TaskSpec: rlarkv1alpha1.TaskSpec{Suspended: true}, + }, "job-worker", "default") + if task.Spec.Suspended { + t.Fatal("Task inherited template suspension without a suspended Job") + } + + job.Spec.Suspended = true + task = buildTask(job, rlarkv1alpha1.JobTaskTemplate{Name: "worker"}, "job-worker", "default") + if !task.Spec.Suspended { + t.Fatal("Task did not inherit Job suspension") + } +} + func TestBuildTaskCarriesJobRestartOperation(t *testing.T) { job := &rlarkv1alpha1.Job{ObjectMeta: metav1.ObjectMeta{ Name: "job", diff --git a/apps/rlark/pkg/gateway/api_reference.go b/apps/rlark/pkg/gateway/api_reference.go index abc8aa8..c79d49c 100644 --- a/apps/rlark/pkg/gateway/api_reference.go +++ b/apps/rlark/pkg/gateway/api_reference.go @@ -83,6 +83,42 @@ func (g *Gateway) handleAPIReference(c *gin.Context) { Description: localizedText{ZH: "查询集群详情", EN: "Get cluster details"}, Example: map[string]any{"data": map[string]any{"id": "cluster-beijing", "name": "Beijing GPU Cluster", "status": "Ready"}}, }, + { + Method: "POST", + Path: "/api/v1/clusters/{cluster_id}/cordon", + Description: localizedText{ZH: "管理员暂停新任务调度到集群", EN: "Administrator: stop scheduling new tasks to a cluster"}, + Example: map[string]any{"success": true}, + }, + { + Method: "POST", + Path: "/api/v1/clusters/{cluster_id}/uncordon", + Description: localizedText{ZH: "管理员恢复集群的新任务调度", EN: "Administrator: resume scheduling new tasks to a cluster"}, + Example: map[string]any{"success": true}, + }, + { + Method: "POST", + Path: "/api/v1/clusters/{cluster_id}/drain", + Description: localizedText{ZH: "管理员排空集群中的 RLark Task", EN: "Administrator: drain RLark Tasks from a cluster"}, + Example: map[string]any{"success": true}, + }, + { + Method: "POST", + Path: "/api/v1/clusters/{cluster_id}/delist", + Description: localizedText{ZH: "管理员下架集群", EN: "Administrator: delist a cluster so it is not shown to users"}, + Example: map[string]any{"success": true}, + }, + { + Method: "POST", + Path: "/api/v1/clusters/{cluster_id}/delete", + Description: localizedText{ZH: "管理员永久删除集群及其数据面资源", EN: "Administrator: permanently delete a cluster and its data-plane resources"}, + Example: map[string]any{"success": true}, + }, + { + Method: "POST", + Path: "/api/v1/clusters/{cluster_id}/restore", + Description: localizedText{ZH: "管理员恢复已下架集群", EN: "Administrator: restore a hidden cluster"}, + Example: map[string]any{"success": true}, + }, }, }, resourceReferenceSection("nodes", "节点", "Nodes", "查询集群中的节点资源；节点写操作仅允许管理员执行。", "List cluster nodes; node write operations require an administrator.", "Node", "NodeList"), diff --git a/apps/rlark/pkg/gateway/auth_handler_test.go b/apps/rlark/pkg/gateway/auth_handler_test.go index 93f16e7..21d21d1 100644 --- a/apps/rlark/pkg/gateway/auth_handler_test.go +++ b/apps/rlark/pkg/gateway/auth_handler_test.go @@ -159,6 +159,10 @@ func TestRegisteredRoutePermissions(t *testing.T) { {name: "shared system config read", method: http.MethodGet, path: "/api/v1/system-config", user: http.StatusOK, admin: http.StatusOK}, {name: "admin system config write", method: http.MethodPut, path: "/api/v1/system-config", user: http.StatusForbidden, admin: http.StatusBadRequest}, {name: "shared ssh keys", method: http.MethodGet, path: "/api/v1/ssh-user-keys", user: http.StatusOK, admin: http.StatusOK}, + {name: "shared storage provider catalog", method: http.MethodGet, path: "/api/v1/storage/storageclass/provider", user: http.StatusOK, admin: http.StatusOK}, + {name: "shared storage class create", method: http.MethodPost, path: "/api/v1/storage/storageclass", user: http.StatusBadRequest, admin: http.StatusBadRequest}, + {name: "shared storage class update", method: http.MethodPut, path: "/api/v1/storage/storageclass/test", user: http.StatusBadRequest, admin: http.StatusBadRequest}, + {name: "shared storage class delete", method: http.MethodDelete, path: "/api/v1/storage/storageclass/test", user: http.StatusNotFound, admin: http.StatusNotFound}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/apps/rlark/pkg/gateway/clusters_handler.go b/apps/rlark/pkg/gateway/clusters_handler.go index 2908a1e..7d4d929 100644 --- a/apps/rlark/pkg/gateway/clusters_handler.go +++ b/apps/rlark/pkg/gateway/clusters_handler.go @@ -1,38 +1,47 @@ package gateway import ( + "encoding/json" + "errors" + "fmt" "net/http" "sort" "strings" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/gin-gonic/gin" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1" ) // ClusterInfo defines the aggregate information returned by the cluster APIs. type ClusterInfo struct { - ID string `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - Region string `json:"region"` - Location string `json:"location"` - Phase rlarkv1alpha1.ClusterPhase `json:"phase"` - TotalNodes int `json:"totalNodes"` - OnlineNodes int `json:"onlineNodes"` - OfflineNodes int `json:"offlineNodes"` - CloudNodes int `json:"cloudNodes"` - EmbodiedNodes int `json:"embodiedNodes"` - Robots int `json:"robots"` - GPUModels []string `json:"gpuModels"` - RobotModels []string `json:"robotModels"` - CPUUsage int `json:"cpuUsage"` - GPUUsage int `json:"gpuUsage"` - RobotUsage int `json:"robotUsage"` - RunningJobs int `json:"runningJobs"` - Description string `json:"description"` + ID string `json:"id"` + AgentNamespace string `json:"agentNamespace"` + Name string `json:"name"` + Type string `json:"type"` + Region string `json:"region"` + Location string `json:"location"` + Phase rlarkv1alpha1.ClusterPhase `json:"phase"` + TotalNodes int `json:"totalNodes"` + OnlineNodes int `json:"onlineNodes"` + OfflineNodes int `json:"offlineNodes"` + CloudNodes int `json:"cloudNodes"` + EmbodiedNodes int `json:"embodiedNodes"` + Robots int `json:"robots"` + GPUModels []string `json:"gpuModels"` + RobotModels []string `json:"robotModels"` + CPUUsage int `json:"cpuUsage"` + GPUUsage int `json:"gpuUsage"` + RobotUsage int `json:"robotUsage"` + RunningJobs int `json:"runningJobs"` + Description string `json:"description"` + AgentNamespaceConflict bool `json:"agentNamespaceConflict,omitempty"` } // ClusterDetail holds details. @@ -78,13 +87,20 @@ func nodeCategories(node rlarkv1alpha1.Node) []rlarkv1alpha1.NodeCategory { return result } +func clusterAgentNamespaceConflict(cluster rlarkv1alpha1.Cluster) bool { + condition := apimeta.FindStatusCondition(cluster.Status.Conditions, rlarkv1alpha1.ConditionTypeValidated) + return condition != nil && condition.Reason == "AgentNamespaceConflict" +} + func buildClusterInfo(cluster rlarkv1alpha1.Cluster, nodes []rlarkv1alpha1.Node) ClusterInfo { info := ClusterInfo{ - ID: cluster.Name, - Name: cluster.Name, - Phase: cluster.Status.Phase, - GPUModels: []string{}, - RobotModels: []string{}, + ID: cluster.Name, + AgentNamespace: cluster.Spec.AgentNamespace, + Name: cluster.Name, + Phase: cluster.Status.Phase, + AgentNamespaceConflict: clusterAgentNamespaceConflict(cluster), + GPUModels: []string{}, + RobotModels: []string{}, } gpuModels := make(map[string]struct{}) robotModels := make(map[string]struct{}) @@ -190,20 +206,219 @@ func (g *Gateway) listClusters(c *gin.Context) { clusters := make([]ClusterInfo, 0, len(clusterList.Items)) for _, cluster := range clusterList.Items { + if cluster.Spec.Delisted && c.GetString(authRoleKey) != "admin" { + continue + } clusters = append(clusters, buildClusterInfo(cluster, grouped[cluster.Spec.AgentNamespace])) } sort.Slice(clusters, func(i, j int) bool { return clusters[i].Name < clusters[j].Name }) c.JSON(http.StatusOK, gin.H{"data": clusters, "success": true}) } -// getCluster handles a Cluster CR detail request. +// updateCluster applies one idempotent administrator lifecycle transition. +func createMergePatch(before, after *rlarkv1alpha1.Cluster) ([]byte, error) { + original, err := json.Marshal(before) + if err != nil { + return nil, fmt.Errorf("marshal original Cluster: %w", err) + } + modified, err := json.Marshal(after) + if err != nil { + return nil, fmt.Errorf("marshal modified Cluster: %w", err) + } + patch, err := jsonpatch.CreateMergePatch(original, modified) + if err != nil { + return nil, fmt.Errorf("create Cluster merge patch: %w", err) + } + return patch, nil +} + +var ( + errClusterDelisted = errors.New("cluster is delisted") + errClusterDraining = errors.New("cluster is draining") +) + +func (g *Gateway) updateCluster(c *gin.Context, update func(*rlarkv1alpha1.Cluster) error) { + if g.kubeClient == nil { + c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "cluster not found"}) + return + } + + clusterID := c.Param("cluster_id") + if clusterID == "" { + respondError(c, http.StatusBadRequest, CodeInvalidRequest, "cluster_id is required") + return + } + var updated *rlarkv1alpha1.Cluster + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cluster, err := g.kubeClient.RlinfV1alpha1().Clusters().Get(c.Request.Context(), clusterID, metav1.GetOptions{}) + if err != nil { + return err + } + before := cluster.DeepCopy() + if err := update(cluster); err != nil { + return err + } + patch, err := createMergePatch(before, cluster) + if err != nil { + return err + } + updated, err = g.kubeClient.RlinfV1alpha1().Clusters().Patch(c.Request.Context(), clusterID, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return err + } + return nil + }) + if err != nil { + if apierrors.IsNotFound(err) { + c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "cluster not found"}) + return + } + switch { + case errors.Is(err, errClusterDelisted), errors.Is(err, errClusterDraining): + respondError(c, http.StatusConflict, CodeResourceConflict, err.Error()) + case apierrors.IsInvalid(err), apierrors.IsBadRequest(err): + respondError(c, http.StatusBadRequest, CodeInvalidRequest, err.Error()) + default: + respondError(c, http.StatusInternalServerError, CodeInternalError, err.Error()) + } + return + } + nodes, err := g.clusterNodes(c) + if err != nil { + respondError(c, http.StatusInternalServerError, CodeInternalError, err.Error()) + return + } + clusterNodes := make([]rlarkv1alpha1.Node, 0) + for _, node := range nodes { + if node.Namespace == updated.Spec.AgentNamespace { + clusterNodes = append(clusterNodes, node) + } + } + c.JSON(http.StatusOK, gin.H{"data": buildClusterInfo(*updated, clusterNodes), "success": true}) +} + +func (g *Gateway) cordonCluster(c *gin.Context) { + g.updateCluster(c, cordonClusterSpec) +} + +func cordonClusterSpec(cluster *rlarkv1alpha1.Cluster) error { + if cluster.Spec.Delisted { + return fmt.Errorf("delisted cluster must be restored before it can be cordoned: %w", errClusterDelisted) + } + cluster.Spec.Cordoned = true + return nil +} + +func (g *Gateway) uncordonCluster(c *gin.Context) { + g.updateCluster(c, uncordonClusterSpec) +} + +func uncordonClusterSpec(cluster *rlarkv1alpha1.Cluster) error { + if cluster.Spec.Delisted { + return fmt.Errorf("delisted cluster must be restored before it can be uncordoned: %w", errClusterDelisted) + } + if cluster.Spec.Draining { + if !drainComplete(cluster) { + return fmt.Errorf("draining cluster must complete draining before it can be uncordoned: %w", errClusterDraining) + } + // Uncordoning a fully drained cluster cancels the drain intent as a whole. + // Leaving Spec.Draining set would keep the cluster unschedulable and let the + // cluster controller keep stopping Tasks, making the uncordon a silent no-op. + cluster.Spec.Draining = false + cluster.Spec.DrainPreviousCordoned = false + } + cluster.Spec.Cordoned = false + return nil +} + +func (g *Gateway) drainCluster(c *gin.Context) { + g.updateCluster(c, drainClusterSpec) +} + +func (g *Gateway) undrainCluster(c *gin.Context) { + g.updateCluster(c, undrainClusterSpec) +} + +func undrainClusterSpec(cluster *rlarkv1alpha1.Cluster) error { + if cluster.Spec.Delisted { + return fmt.Errorf("delisted cluster must be restored before its drain can be cancelled: %w", errClusterDelisted) + } + cluster.Spec.Draining = false + cluster.Spec.Cordoned = cluster.Spec.DrainPreviousCordoned + cluster.Spec.DrainPreviousCordoned = false + return nil +} + +func drainClusterSpec(cluster *rlarkv1alpha1.Cluster) error { + if cluster.Spec.Delisted { + return fmt.Errorf("delisted cluster must be restored before it can be drained: %w", errClusterDelisted) + } + if !cluster.Spec.Draining { + cluster.Spec.DrainPreviousCordoned = cluster.Spec.Cordoned + } + cluster.Spec.Cordoned = true + cluster.Spec.Draining = true + return nil +} + +func (g *Gateway) delistCluster(c *gin.Context) { + g.updateCluster(c, delistClusterSpec) +} + +func drainComplete(cluster *rlarkv1alpha1.Cluster) bool { + condition := apimeta.FindStatusCondition(cluster.Status.Conditions, rlarkv1alpha1.ConditionTypeProgressing) + return condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == "DrainComplete" && condition.ObservedGeneration == cluster.Generation +} + +func delistClusterSpec(cluster *rlarkv1alpha1.Cluster) error { + if cluster.Spec.Draining && !drainComplete(cluster) { + return fmt.Errorf("cluster drain must complete before it can be delisted: %w", errClusterDraining) + } + cluster.Spec.Delisted = true + return nil +} + +func (g *Gateway) deleteCluster(c *gin.Context) { + if g.kubeClient == nil { + c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "cluster not found"}) + return + } + + clusterID := c.Param("cluster_id") + if clusterID == "" { + respondError(c, http.StatusBadRequest, CodeInvalidRequest, "cluster_id is required") + return + } + if err := g.kubeClient.RlinfV1alpha1().Clusters().Delete(c.Request.Context(), clusterID, metav1.DeleteOptions{}); err != nil { + if apierrors.IsNotFound(err) { + c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "cluster not found"}) + return + } + respondError(c, http.StatusInternalServerError, CodeInternalError, err.Error()) + return + } + c.JSON(http.StatusOK, gin.H{"success": true}) +} + +func (g *Gateway) restoreCluster(c *gin.Context) { + g.updateCluster(c, func(cluster *rlarkv1alpha1.Cluster) error { + cluster.Spec.Delisted = false + return nil + }) +} + func (g *Gateway) getCluster(c *gin.Context) { if g.kubeClient == nil { c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "cluster not found"}) return } - cluster, err := g.kubeClient.RlinfV1alpha1().Clusters().Get(c.Request.Context(), c.Param("cluster_id"), metav1.GetOptions{}) + clusterID := c.Param("cluster_id") + if clusterID == "" { + respondError(c, http.StatusBadRequest, CodeInvalidRequest, "cluster_id is required") + return + } + cluster, err := g.kubeClient.RlinfV1alpha1().Clusters().Get(c.Request.Context(), clusterID, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "cluster not found"}) @@ -218,6 +433,11 @@ func (g *Gateway) getCluster(c *gin.Context) { return } + if cluster.Spec.Delisted && c.GetString(authRoleKey) != "admin" { + c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "cluster not found"}) + return + } + clusterNodes := make([]rlarkv1alpha1.Node, 0) for _, node := range nodes { if node.Namespace == cluster.Spec.AgentNamespace { diff --git a/apps/rlark/pkg/gateway/clusters_handler_test.go b/apps/rlark/pkg/gateway/clusters_handler_test.go index a61acd4..b5620e4 100644 --- a/apps/rlark/pkg/gateway/clusters_handler_test.go +++ b/apps/rlark/pkg/gateway/clusters_handler_test.go @@ -1,13 +1,331 @@ package gateway import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/gin-gonic/gin" + clientset "github.com/rlinf/rlark/api/kubeclients/clientset/versioned/fake" rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1" ) +func TestClusterTransitionErrors(t *testing.T) { + t.Parallel() + tests := []struct { + name string + update func(*rlarkv1alpha1.Cluster) error + cluster rlarkv1alpha1.Cluster + wantErr error + }{ + { + name: "cordon rejects delisted cluster", + update: cordonClusterSpec, + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Delisted: true}}, + wantErr: errClusterDelisted, + }, + { + name: "uncordon rejects incomplete drain", + update: uncordonClusterSpec, + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Draining: true}}, + wantErr: errClusterDraining, + }, + { + name: "uncordon accepts completed drain", + update: uncordonClusterSpec, + cluster: rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Spec: rlarkv1alpha1.ClusterSpec{Draining: true, Cordoned: true}, + Status: rlarkv1alpha1.ClusterStatus{Conditions: []metav1.Condition{{ + Type: rlarkv1alpha1.ConditionTypeProgressing, + Status: metav1.ConditionFalse, + Reason: "DrainComplete", + ObservedGeneration: 1, + }}}, + }, + }, + { + name: "drain rejects delisted cluster", + update: drainClusterSpec, + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Delisted: true}}, + wantErr: errClusterDelisted, + }, + { + name: "undrain rejects delisted cluster", + update: undrainClusterSpec, + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Delisted: true}}, + wantErr: errClusterDelisted, + }, + { + name: "delist rejects incomplete drain", + update: delistClusterSpec, + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Draining: true}}, + wantErr: errClusterDraining, + }, + { + name: "delist accepts completed drain", + update: delistClusterSpec, + cluster: rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Spec: rlarkv1alpha1.ClusterSpec{Draining: true, Cordoned: true}, + Status: rlarkv1alpha1.ClusterStatus{Conditions: []metav1.Condition{{ + Type: rlarkv1alpha1.ConditionTypeProgressing, + Status: metav1.ConditionFalse, + Reason: "DrainComplete", + ObservedGeneration: 1, + }}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.update(tt.cluster.DeepCopy()) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want errors.Is(..., %v)", err, tt.wantErr) + } + }) + } +} + +func TestClusterLifecycleBusinessErrorCodes(t *testing.T) { + gin.SetMode(gin.TestMode) + tests := []struct { + name string + path string + cluster *rlarkv1alpha1.Cluster + wantStatus int + wantCode string + }{ + { + name: "drain rejects delisted cluster", + path: "/clusters/cluster-a/drain", + cluster: &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, Spec: rlarkv1alpha1.ClusterSpec{Delisted: true}}, + wantStatus: http.StatusConflict, + wantCode: CodeResourceConflict, + }, + { + name: "undrain rejects delisted cluster", + path: "/clusters/cluster-a/undrain", + cluster: &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, Spec: rlarkv1alpha1.ClusterSpec{Delisted: true, Draining: true}}, + wantStatus: http.StatusConflict, + wantCode: CodeResourceConflict, + }, + { + name: "delist rejects draining cluster", + path: "/clusters/cluster-a/delist", + cluster: &rlarkv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, Spec: rlarkv1alpha1.ClusterSpec{Draining: true}}, + wantStatus: http.StatusConflict, + wantCode: CodeResourceConflict, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gateway := &Gateway{kubeClient: clientset.NewSimpleClientset(tt.cluster)} + router := gin.New() + router.POST("/clusters/:cluster_id/drain", gateway.drainCluster) + router.POST("/clusters/:cluster_id/undrain", gateway.undrainCluster) + router.POST("/clusters/:cluster_id/uncordon", gateway.uncordonCluster) + router.POST("/clusters/:cluster_id/delist", gateway.delistCluster) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, tt.path, nil)) + + if recorder.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", recorder.Code, tt.wantStatus) + } + var response ErrorResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode error response: %v", err) + } + if response.Code != tt.wantCode { + t.Fatalf("error code = %q, want %q", response.Code, tt.wantCode) + } + }) + } +} + +func TestClusterLifecycleResponseIncludesClusterNodes(t *testing.T) { + gin.SetMode(gin.TestMode) + cluster := &rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "rlark-agent-a"}, + } + node := &rlarkv1alpha1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-a", + Namespace: "rlark-agent-a", + Labels: map[string]string{ + rlarkv1alpha1.LabelNodeCategory + "-cloud": "true", + }, + }, + Status: rlarkv1alpha1.NodeStatus{Phase: rlarkv1alpha1.NodeOnline}, + } + gateway := &Gateway{kubeClient: clientset.NewSimpleClientset(cluster, node)} + router := gin.New() + router.POST("/clusters/:cluster_id/cordon", gateway.cordonCluster) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/clusters/cluster-a/cordon", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK) + } + var response struct { + Data ClusterInfo `json:"data"` + Success bool `json:"success"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode lifecycle response: %v", err) + } + if !response.Success || response.Data.TotalNodes != 1 || response.Data.OnlineNodes != 1 || response.Data.CloudNodes != 1 || response.Data.Type != "Cloud" { + t.Fatalf("unexpected lifecycle response: %#v", response) + } +} + +func TestDelistClusterTransition(t *testing.T) { + cluster := &rlarkv1alpha1.Cluster{} + if err := delistClusterSpec(cluster); err != nil { + t.Fatalf("delistClusterSpec() error = %v", err) + } + if !cluster.Spec.Delisted { + t.Fatal("expected Delisted to be true") + } +} + +func TestDrainClusterTransition(t *testing.T) { + tests := []struct { + name string + cluster rlarkv1alpha1.Cluster + wantErr string + wantCordoned bool + wantPrevious bool + }{ + { + name: "cordons and saves uncordoned state", + wantCordoned: true, + }, + { + name: "preserves previously cordoned state", + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Cordoned: true}}, + wantCordoned: true, + wantPrevious: true, + }, + { + name: "does not overwrite state during repeated drain", + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Cordoned: true, Draining: true}}, + wantCordoned: true, + }, + { + name: "rejects delisted cluster", + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Delisted: true}}, + wantErr: "delisted cluster must be restored before it can be drained", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cluster := tt.cluster.DeepCopy() + err := drainClusterSpec(cluster) + if tt.wantErr != "" { + if err == nil || !strings.HasPrefix(err.Error(), tt.wantErr) { + t.Fatalf("drainClusterSpec() error = %v, want prefix %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("drainClusterSpec() error = %v", err) + } + if !cluster.Spec.Draining || cluster.Spec.Cordoned != tt.wantCordoned || cluster.Spec.DrainPreviousCordoned != tt.wantPrevious { + t.Fatalf("unexpected drain state: %#v", cluster.Spec) + } + }) + } +} + +func TestUndrainClusterTransition(t *testing.T) { + cluster := &rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{ + Cordoned: true, + Draining: true, + DrainPreviousCordoned: false, + }} + if err := undrainClusterSpec(cluster); err != nil { + t.Fatalf("undrainClusterSpec() error = %v", err) + } + if cluster.Spec.Draining || cluster.Spec.Cordoned || cluster.Spec.DrainPreviousCordoned { + t.Fatalf("unexpected undrain state: %#v", cluster.Spec) + } +} + +func TestUncordonClusterTransition(t *testing.T) { + tests := []struct { + name string + cluster rlarkv1alpha1.Cluster + wantErr string + }{ + { + name: "clears cordon", + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Cordoned: true}}, + }, + { + name: "rejects delisted cluster", + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Delisted: true, Cordoned: true}}, + wantErr: "delisted cluster must be restored before it can be uncordoned", + }, + { + name: "rejects draining cluster", + cluster: rlarkv1alpha1.Cluster{Spec: rlarkv1alpha1.ClusterSpec{Draining: true, Cordoned: true}}, + wantErr: "draining cluster must complete draining before it can be uncordoned", + }, + { + name: "cancels completed drain", + cluster: rlarkv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Generation: 3}, + Spec: rlarkv1alpha1.ClusterSpec{ + Draining: true, + Cordoned: true, + DrainPreviousCordoned: true, + }, + Status: rlarkv1alpha1.ClusterStatus{Conditions: []metav1.Condition{{ + Type: rlarkv1alpha1.ConditionTypeProgressing, + Status: metav1.ConditionFalse, + Reason: "DrainComplete", + ObservedGeneration: 3, + }}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cluster := tt.cluster.DeepCopy() + err := uncordonClusterSpec(cluster) + if tt.wantErr != "" { + if err == nil || !strings.HasPrefix(err.Error(), tt.wantErr) { + t.Fatalf("uncordonClusterSpec() error = %v, want prefix %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("uncordonClusterSpec() error = %v", err) + } + if cluster.Spec.Cordoned { + t.Fatal("expected Cordoned to be false") + } + if cluster.Spec.Draining { + t.Fatal("expected Draining to be false so the cluster becomes schedulable again") + } + if cluster.Spec.DrainPreviousCordoned { + t.Fatal("expected DrainPreviousCordoned to be cleared with the drain intent") + } + }) + } +} + func TestBuildClusterInfoUsesCityMetadata(t *testing.T) { nodes := []rlarkv1alpha1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -78,6 +396,7 @@ func TestBuildClusterInfoBooleanCategoryLabel(t *testing.T) { func TestBuildClusterInfoUsesClusterStatus(t *testing.T) { cluster := rlarkv1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}, + Spec: rlarkv1alpha1.ClusterSpec{AgentNamespace: "rlark-agent-a"}, Status: rlarkv1alpha1.ClusterStatus{Phase: rlarkv1alpha1.ClusterDowngraded}, } @@ -85,6 +404,9 @@ func TestBuildClusterInfoUsesClusterStatus(t *testing.T) { if info.Phase != rlarkv1alpha1.ClusterDowngraded { t.Fatalf("expected Downgraded, got %q", info.Phase) } + if info.AgentNamespace != "rlark-agent-a" { + t.Fatalf("agent namespace = %q, want %q", info.AgentNamespace, "rlark-agent-a") + } } func TestBuildClusterInfoMixedCategoryLabelForms(t *testing.T) { diff --git a/apps/rlark/pkg/gateway/kube_handler.go b/apps/rlark/pkg/gateway/kube_handler.go index 2e59654..e965878 100644 --- a/apps/rlark/pkg/gateway/kube_handler.go +++ b/apps/rlark/pkg/gateway/kube_handler.go @@ -268,7 +268,7 @@ func (g *Gateway) handleListNodes(c *gin.Context) { respondKubeError(c, err.Error(), err) return } - clusterNamespaces := uniqueClusterNamespaces(c.Request.Context(), clusters.Items) + clusterNamespaces := uniqueClusterNamespaces(c.Request.Context(), visibleClusterNamespaces(clusters.Items, c.GetString(authRoleKey) == "admin")) requestedNamespace := c.Query("namespace") if requestedNamespace != "" { @@ -339,6 +339,20 @@ func (g *Gateway) listAggregatedNodes(ctx context.Context, namespaces map[string return items, hasMore, nil } +func visibleClusterNamespaces(clusters []rlarkiov1alpha1.Cluster, includeDelisted bool) []rlarkiov1alpha1.Cluster { + if includeDelisted { + return clusters + } + + visible := make([]rlarkiov1alpha1.Cluster, 0, len(clusters)) + for _, cluster := range clusters { + if !cluster.Spec.Delisted { + visible = append(visible, cluster) + } + } + return visible +} + func uniqueClusterNamespaces(ctx context.Context, clusters []rlarkiov1alpha1.Cluster) map[string]struct{} { namespaces := make(map[string]string, len(clusters)) for _, cluster := range clusters { @@ -802,6 +816,22 @@ func isTenantResource(resource string) bool { return resource == "workflows" || resource == "jobs" || resource == "tasks" } +func (g *Gateway) validateTaskCreateNamespace(ctx context.Context, namespace string) error { + clusters, err := g.kubeClient.RlinfV1alpha1().Clusters().List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list Clusters for Task namespace %q: %w", namespace, err) + } + for _, cluster := range clusters.Items { + if cluster.Spec.AgentNamespace != namespace { + continue + } + if cluster.Spec.Cordoned || cluster.Spec.Draining || cluster.Spec.Delisted { + return fmt.Errorf("cluster %q does not accept new tasks", cluster.Name) + } + } + return nil +} + func (g *Gateway) handleKubeCreate(resource string) gin.HandlerFunc { return func(c *gin.Context) { a, ok := g.accessors[resource] @@ -828,6 +858,12 @@ func (g *Gateway) handleKubeCreate(resource string) gin.HandlerFunc { return } } + if resource == "tasks" { + if err := g.validateTaskCreateNamespace(ctx, namespace); err != nil { + respondError(c, http.StatusConflict, CodeResourceConflict, err.Error()) + return + } + } if isTenantResource(resource) { scope, ok := g.tenantScopeFor(c, resource) if !ok { @@ -931,7 +967,6 @@ func (g *Gateway) handleKubeUpdate(resource string) gin.HandlerFunc { if !enforceSpecQuotas(c, resource, bodyBytes) { return } - result, err := a.update(ctx, namespace, name, bodyBytes) if err != nil { respondCreateError(c, err) @@ -985,7 +1020,6 @@ func (g *Gateway) handleKubePatch(resource string) gin.HandlerFunc { if !enforceSpecQuotas(c, resource, bodyBytes) { return } - result, err := a.patch(ctx, namespace, name, bodyBytes) if err != nil { respondCreateError(c, err) diff --git a/apps/rlark/pkg/gateway/kube_handler_test.go b/apps/rlark/pkg/gateway/kube_handler_test.go index 4181272..201bb04 100644 --- a/apps/rlark/pkg/gateway/kube_handler_test.go +++ b/apps/rlark/pkg/gateway/kube_handler_test.go @@ -11,6 +11,7 @@ import ( "regexp" "slices" "strconv" + "strings" "testing" "github.com/gin-gonic/gin" @@ -21,6 +22,90 @@ import ( "github.com/rlinf/rlark/apps/rlark/pkg/tenant" ) +func TestValidateTaskCreateNamespaceRejectsIneligibleClusters(t *testing.T) { + for _, tt := range []struct { + name string + cluster rlarkiov1alpha1.Cluster + }{ + {name: "cordoned", cluster: rlarkiov1alpha1.Cluster{Spec: rlarkiov1alpha1.ClusterSpec{Cordoned: true}}}, + {name: "draining", cluster: rlarkiov1alpha1.Cluster{Spec: rlarkiov1alpha1.ClusterSpec{Draining: true}}}, + {name: "delisted", cluster: rlarkiov1alpha1.Cluster{Spec: rlarkiov1alpha1.ClusterSpec{Delisted: true}}}, + } { + t.Run(tt.name, func(t *testing.T) { + tt.cluster.Name = "cluster" + tt.cluster.Spec.AgentNamespace = "agent" + g := &Gateway{kubeClient: clientset.NewSimpleClientset(&tt.cluster)} + if err := g.validateTaskCreateNamespace(context.Background(), "agent"); err == nil { + t.Fatal("expected Task create rejection") + } + }) + } +} + +func TestValidateTaskCreateNamespaceAllowsEligibleCluster(t *testing.T) { + cluster := &rlarkiov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Spec: rlarkiov1alpha1.ClusterSpec{AgentNamespace: "agent"}, + } + g := &Gateway{kubeClient: clientset.NewSimpleClientset(cluster)} + if err := g.validateTaskCreateNamespace(context.Background(), "agent"); err != nil { + t.Fatal(err) + } +} + +func TestTaskCreateRejectsIneligibleClusterButExistingTaskWritesAreAllowed(t *testing.T) { + for _, tt := range []struct { + name string + method string + path string + body string + wantStatus int + }{ + {name: "create", method: http.MethodPost, path: "/tasks?namespace=agent", body: `{"metadata":{"name":"task","namespace":"agent"}}`, wantStatus: http.StatusConflict}, + {name: "update", method: http.MethodPut, path: "/tasks/task?namespace=agent", body: `{"metadata":{"name":"task","namespace":"agent"}}`, wantStatus: http.StatusOK}, + {name: "patch", method: http.MethodPatch, path: "/tasks/task?namespace=agent", body: `{"spec":{"suspended":false}}`, wantStatus: http.StatusOK}, + } { + t.Run(tt.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + client := clientset.NewSimpleClientset( + &rlarkiov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Spec: rlarkiov1alpha1.ClusterSpec{AgentNamespace: "agent", Cordoned: true}, + }, + &rlarkiov1alpha1.Task{ObjectMeta: metav1.ObjectMeta{Name: "task", Namespace: "agent"}}, + ) + g := &Gateway{kubeClient: client, accessors: registerAccessors(client)} + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Params = gin.Params{{Key: "name", Value: "task"}} + ctx.Set(authRoleKey, roleAdmin) + ctx.Request = httptest.NewRequest(tt.method, tt.path, strings.NewReader(tt.body)) + + switch tt.method { + case http.MethodPost: + g.handleKubeCreate("tasks")(ctx) + case http.MethodPut: + g.handleKubeUpdate("tasks")(ctx) + case http.MethodPatch: + g.handleKubePatch("tasks")(ctx) + } + + if recorder.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d: %s", recorder.Code, tt.wantStatus, recorder.Body.String()) + } + if tt.wantStatus == http.StatusConflict { + var response ErrorResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Code != CodeResourceConflict { + t.Fatalf("code = %q, want %q", response.Code, CodeResourceConflict) + } + } + }) + } +} + func TestPrepareJobForCreateGeneratesResourceName(t *testing.T) { body := []byte(`{"apiVersion":"rlinf.io/v1alpha1","kind":"Job","metadata":{"name":"Readable Job"}}`) @@ -171,6 +256,68 @@ func TestHandleListNodesFiltersNamespacesNotOwnedByClusters(t *testing.T) { } } +func TestHandleListNodesHidesDelistedClusterNodesFromNonAdmins(t *testing.T) { + gin.SetMode(gin.TestMode) + client := clientset.NewSimpleClientset( + &rlarkiov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "active-cluster"}, + Spec: rlarkiov1alpha1.ClusterSpec{AgentNamespace: "active-agent"}, + }, + &rlarkiov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "delisted-cluster"}, + Spec: rlarkiov1alpha1.ClusterSpec{AgentNamespace: "delisted-agent", Delisted: true}, + }, + &rlarkiov1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "active-node", Namespace: "active-agent"}}, + &rlarkiov1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "delisted-node", Namespace: "delisted-agent"}}, + ) + g := &Gateway{kubeClient: client, accessors: registerAccessors(client)} + + response := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(response) + ctx.Request = httptest.NewRequest(http.MethodGet, "/nodes", nil) + g.handleListNodes(ctx) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) + } + var result rlarkiov1alpha1.NodeList + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + if len(result.Items) != 1 || result.Items[0].Name != "active-node" { + t.Fatalf("items = %#v, want only active-node", result.Items) + } +} + +func TestHandleListNodesShowsDelistedClusterNodesToAdmins(t *testing.T) { + gin.SetMode(gin.TestMode) + client := clientset.NewSimpleClientset( + &rlarkiov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "delisted-cluster"}, + Spec: rlarkiov1alpha1.ClusterSpec{AgentNamespace: "delisted-agent", Delisted: true}, + }, + &rlarkiov1alpha1.Node{ObjectMeta: metav1.ObjectMeta{Name: "delisted-node", Namespace: "delisted-agent"}}, + ) + g := &Gateway{kubeClient: client, accessors: registerAccessors(client)} + + response := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(response) + ctx.Set(authRoleKey, "admin") + ctx.Request = httptest.NewRequest(http.MethodGet, "/nodes", nil) + g.handleListNodes(ctx) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) + } + var result rlarkiov1alpha1.NodeList + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + if len(result.Items) != 1 || result.Items[0].Name != "delisted-node" { + t.Fatalf("items = %#v, want delisted-node", result.Items) + } +} + func TestHandleListNodesExcludesAnExplicitOrphanNamespace(t *testing.T) { gin.SetMode(gin.TestMode) client := clientset.NewSimpleClientset( @@ -646,3 +793,54 @@ func TestWithRestoredTypeMetaKeepsExistingKindAndUnknownValues(t *testing.T) { t.Errorf("restored = %#v, want the non-Kubernetes value unchanged", restored) } } + +// TestParseTagSelectorsGroupsPairsByKey pins the grouping that turns the flat +// query parameter into the per-key value sets the matcher consumes. +func TestParseTagSelectorsGroupsPairsByKey(t *testing.T) { + selectors := parseTagSelectors("env=prod,env=staging,team=infra,broken,=v,k=") + if len(selectors) != 2 { + t.Fatalf("selectors = %#v, want only the two well-formed keys", selectors) + } + if len(selectors["env"]) != 2 { + t.Errorf("env values = %#v, want prod and staging", selectors["env"]) + } + if _, ok := selectors["team"]["infra"]; !ok { + t.Errorf("team values = %#v, want infra", selectors["team"]) + } +} + +// TestJobMatchesTagSelectorsAndsKeysAndOrsValues pins the filter semantics the +// console surfaces in the tag filter popover: selecting several values of one +// key widens the result, while selecting several keys narrows it. +func TestJobMatchesTagSelectorsAndsKeysAndOrsValues(t *testing.T) { + job := rlarkiov1alpha1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "j1"}, + Spec: rlarkiov1alpha1.JobSpec{ + Tags: []rlarkiov1alpha1.JobTag{ + {Key: "env", Values: []string{"prod"}}, + {Key: "team", Values: []string{"infra", "platform"}}, + }, + }, + } + + for _, tc := range []struct { + name string + raw string + match bool + }{ + {"no selector keeps everything", "", true}, + {"single pair hits", "env=prod", true}, + {"unknown value misses", "env=dev", false}, + {"values of one key are ORed", "env=dev,env=prod", true}, + {"every value of one key may hit", "team=infra,team=platform", true}, + {"different keys are ANDed", "env=prod,team=infra", true}, + {"a missing key fails the whole selector", "env=prod,region=cn", false}, + {"a wrong value of one key fails the whole selector", "env=prod,team=qa", false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := jobMatchesTagSelectors(job, parseTagSelectors(tc.raw)); got != tc.match { + t.Errorf("jobMatchesTagSelectors(%q) = %t, want %t", tc.raw, got, tc.match) + } + }) + } +} diff --git a/apps/rlark/pkg/gateway/router.go b/apps/rlark/pkg/gateway/router.go index 1ef0a66..a82af55 100644 --- a/apps/rlark/pkg/gateway/router.go +++ b/apps/rlark/pkg/gateway/router.go @@ -38,6 +38,16 @@ func (g *Gateway) RegisterRoutes(r gin.IRouter) { clusters.GET("", g.listClusters) clusters.GET("/:cluster_id", g.getCluster) } + adminClusters := adminAPI.Group("/clusters") + { + adminClusters.POST("/:cluster_id/cordon", g.cordonCluster) + adminClusters.POST("/:cluster_id/uncordon", g.uncordonCluster) + adminClusters.POST("/:cluster_id/drain", g.drainCluster) + adminClusters.POST("/:cluster_id/undrain", g.undrainCluster) + adminClusters.POST("/:cluster_id/delist", g.delistCluster) + adminClusters.POST("/:cluster_id/delete", g.deleteCluster) + adminClusters.POST("/:cluster_id/restore", g.restoreCluster) + } nodes := rlinfv1alpha1.Group("/nodes") { @@ -153,8 +163,12 @@ func (g *Gateway) RegisterRoutes(r gin.IRouter) { storage := api.Group("/storage") { storage.GET("/storageclass", g.listStorageClass) + storage.GET("/storageclass/provider", g.listProvider) + storage.POST("/storageclass", g.createStorageClass) + storage.PUT("/storageclass/:name", g.updateStorageClass) + storage.DELETE("/storageclass/:name", g.deleteStorageClass) - scFiles := storage.Group("/storageclass/:name/:cluster") + scFiles := storage.Group("/storageclass/:name") { scFiles.GET("/list", g.listStorageClassFiles) scFiles.POST("/upload", g.uploadStorageClassFile) @@ -162,14 +176,6 @@ func (g *Gateway) RegisterRoutes(r gin.IRouter) { scFiles.DELETE("/object/*key", g.deleteStorageClassObject) } } - adminStorage := adminAPI.Group("/storage") - { - adminStorage.POST("/storageclass", g.createStorageClass) - adminStorage.PUT("/storageclass/:name", g.updateStorageClass) - adminStorage.DELETE("/storageclass/:name", g.deleteStorageClass) - adminStorage.GET("/storageclass/provider", g.listProvider) - } - // Addon Catalog APIs addons := adminAPI.Group("/addons") { diff --git a/apps/rlark/pkg/gateway/storage/bucket.go b/apps/rlark/pkg/gateway/storage/bucket.go index 054bfca..306d6dd 100644 --- a/apps/rlark/pkg/gateway/storage/bucket.go +++ b/apps/rlark/pkg/gateway/storage/bucket.go @@ -8,14 +8,18 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" ) -// IsBucketExist reports whether bucketExist. -func (c *Client) IsBucketExist() (bool, error) { - ctx := context.Background() +// ValidateBucketCredentials verifies that the configured credentials can access +// the configured bucket. +func (c *Client) ValidateBucketCredentials(ctx context.Context) error { _, err := c.s3Client.HeadBucket(ctx, &s3.HeadBucketInput{ Bucket: aws.String(c.getBucket()), }) - if err != nil { - // Check if bucket not found + return err +} + +// IsBucketExist reports whether bucketExist. +func (c *Client) IsBucketExist() (bool, error) { + if err := c.ValidateBucketCredentials(context.Background()); err != nil { return false, err } return true, nil diff --git a/apps/rlark/pkg/gateway/storage/bucket_test.go b/apps/rlark/pkg/gateway/storage/bucket_test.go new file mode 100644 index 0000000..a5f53a3 --- /dev/null +++ b/apps/rlark/pkg/gateway/storage/bucket_test.go @@ -0,0 +1,48 @@ +package storage + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateBucketCredentials(t *testing.T) { + cases := []struct { + name string + statusCode int + wantErr bool + }{ + {name: "accepts accessible bucket", statusCode: http.StatusOK}, + {name: "returns access denied", statusCode: http.StatusForbidden, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var method, path string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method = r.Method + path = r.URL.Path + w.WriteHeader(tc.statusCode) + })) + defer server.Close() + + client, err := NewClient(&Config{ + AccessKeyId: "access", SecretAccessKey: "secret", Bucket: "bucket", + Endpoint: server.URL, Region: "us-east-1", UsePathStyle: true, Provider: "Minio", + }) + require.NoError(t, err) + + err = client.ValidateBucketCredentials(context.Background()) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, http.MethodHead, method) + assert.Equal(t, "/bucket", path) + }) + } +} diff --git a/apps/rlark/pkg/gateway/storage/client.go b/apps/rlark/pkg/gateway/storage/client.go index a92fb61..3349f96 100644 --- a/apps/rlark/pkg/gateway/storage/client.go +++ b/apps/rlark/pkg/gateway/storage/client.go @@ -2,14 +2,12 @@ package storage import ( "context" - "strings" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/sirupsen/logrus" ) // Client is a client. @@ -34,37 +32,20 @@ func NewClient(config *Config) (*Client, error) { } cfg.BaseEndpoint = aws.String(config.Endpoint) - // cfg.BaseEndpoint = aws.String(fmt.Sprintf("https://%s.oss-%s.aliyuncs.com", config.Bucket, config.Region)) - // } else if config.Endpoint != "" { - // cfg.BaseEndpoint = aws.String(config.Endpoint) - // } client := s3.NewFromConfig(cfg, func(o *s3.Options) { - o.UsePathStyle = config.UsePathStyle - if strings.EqualFold(config.Provider, "Alibaba") { - o.UsePathStyle = false - } + // 寻址方式与下发到数据面的 rclone 配置共用同一判定, 避免同一条存储记录 + // 在网关直连与挂载链路上使用不同的 URL 形式. + o.UsePathStyle = EffectivePathStyle(config.Provider, config.UsePathStyle) o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired }) - ossClient := &Client{ + return &Client{ config: config, s3Client: client, presignClient: s3.NewPresignClient(client), - } - - go func() { - logrus.Info("S3 client warming up...") - _, err := ossClient.IsBucketExist() - if err != nil { - logrus.Warnf("S3 connection warmup failed: %v", err) - } else { - logrus.Info("S3 connection warmed up successfully") - } - }() - - return ossClient, nil + }, nil } // NewClientWithDefault creates a new ClientWithDefault. diff --git a/apps/rlark/pkg/gateway/storage/provider.go b/apps/rlark/pkg/gateway/storage/provider.go new file mode 100644 index 0000000..79d90d4 --- /dev/null +++ b/apps/rlark/pkg/gateway/storage/provider.go @@ -0,0 +1,138 @@ +package storage + +import ( + "fmt" + "net/url" + "strings" +) + +// Provider describes one selectable S3-compatible backend. +// +// Value is the canonical rclone `provider` setting written into the +// StorageClass Secret. rclone does not fail on an unknown provider: it logs a +// warning and silently falls back to its generic "Other" profile, so a wrong +// value is only visible as a mount-time I/O error on the data plane. Every +// value accepted by the API must therefore come from this catalog. +type Provider struct { + // DisplayName is the human-readable label presented to clients. + DisplayName string + // Value is the canonical rclone s3 provider value. + Value string + // Aliases are additional spellings accepted on input, kept so that records + // written by older clients can still be read and rewritten canonically. + Aliases []string + // VirtualHostedOnly reports that the backend rejects path-style addressing. + VirtualHostedOnly bool + // RegionEndpointFormat derives the endpoint from the region when the caller + // leaves the endpoint empty. An empty format means the endpoint is required. + RegionEndpointFormat string +} + +// Providers is the ordered catalog of selectable providers. Several entries +// intentionally share the rclone value "Other", so the catalog is an ordered +// slice rather than a map keyed by value. +var Providers = []Provider{ + {DisplayName: "AWS S3", Value: "AWS", Aliases: []string{"Amazon S3"}}, + { + DisplayName: "阿里云 OSS", + Value: "Alibaba", + Aliases: []string{"Aliyun OSS", "Aliyun", "AlibabaOSS"}, + VirtualHostedOnly: true, + RegionEndpointFormat: "https://oss-%s.aliyuncs.com", + }, + {DisplayName: "腾讯云 COS", Value: "TencentCOS", Aliases: []string{"Tencent COS"}, VirtualHostedOnly: true}, + {DisplayName: "华为云 OBS", Value: "HuaweiOBS", Aliases: []string{"Huawei OBS"}, VirtualHostedOnly: true}, + {DisplayName: "七牛云 Kodo", Value: "Qiniu"}, + {DisplayName: "移动云 EOS", Value: "ChinaMobile"}, + {DisplayName: "网易 NOS", Value: "Netease"}, + {DisplayName: "UCloud US3", Value: "Ucloud"}, + {DisplayName: "MinIO", Value: "Minio"}, + {DisplayName: "Ceph RGW", Value: "Ceph"}, + {DisplayName: "SeaweedFS", Value: "SeaweedFS"}, + {DisplayName: "Cloudflare R2", Value: "Cloudflare"}, + {DisplayName: "Google Cloud Storage", Value: "GCS", Aliases: []string{"Google Cloud"}}, + {DisplayName: "DigitalOcean Spaces", Value: "DigitalOcean"}, + {DisplayName: "Wasabi", Value: "Wasabi"}, + {DisplayName: "Backblaze B2 (S3)", Value: "Other"}, + {DisplayName: "IBM COS", Value: "IBMCOS"}, + {DisplayName: "Scaleway", Value: "Scaleway"}, + {DisplayName: "OVHcloud", Value: "OVH"}, + {DisplayName: "Linode / Akamai", Value: "Linode"}, + {DisplayName: "IDrive e2", Value: "IDrive"}, + {DisplayName: "Seagate Lyve Cloud", Value: "LyveCloud"}, + {DisplayName: "Storj (S3 网关)", Value: "Storj"}, + {DisplayName: "Synology C2", Value: "Synology"}, + {DisplayName: "ArvanCloud", Value: "ArvanCloud"}, + {DisplayName: "Hetzner", Value: "Hetzner"}, + {DisplayName: "IONOS", Value: "IONOS"}, + {DisplayName: "MEGA S4", Value: "Mega"}, + {DisplayName: "RackCorp", Value: "RackCorp"}, + {DisplayName: "Dreamhost", Value: "Dreamhost"}, + {DisplayName: "其他 S3 兼容", Value: "Other"}, +} + +// LookupProvider resolves a provider spelling to its catalog entry. The canonical +// value, the display name, and the recorded aliases are all accepted so that +// records written by older clients remain readable. +func LookupProvider(provider string) (Provider, bool) { + provider = strings.TrimSpace(provider) + if provider == "" { + return Provider{}, false + } + for _, candidate := range Providers { + if strings.EqualFold(candidate.Value, provider) || strings.EqualFold(candidate.DisplayName, provider) { + return candidate, true + } + for _, alias := range candidate.Aliases { + if strings.EqualFold(alias, provider) { + return candidate, true + } + } + } + return Provider{}, false +} + +// EffectivePathStyle reports the addressing style to use for provider. Backends +// that only answer virtual-hosted requests always force path-style off, so the +// gateway client and the rclone config rendered for the data plane agree even +// when a stored record still carries the requested value. +func EffectivePathStyle(provider string, requested bool) bool { + if entry, ok := LookupProvider(provider); ok && entry.VirtualHostedOnly { + return false + } + return requested +} + +// NormalizeEndpoint returns the canonical endpoint URL for provider. +// +// The result never embeds the bucket name: both the AWS SDK and rclone compose +// virtual-hosted URLs themselves, so a pre-composed endpoint is addressed as +// bucket.bucket.. An empty endpoint is derived from the region when the +// provider publishes a regional endpoint format. +func NormalizeEndpoint(provider, endpoint, region string) (string, error) { + entry, _ := LookupProvider(provider) + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + if entry.RegionEndpointFormat == "" { + return "", fmt.Errorf("endpoint is required") + } + if strings.TrimSpace(region) == "" { + return "", fmt.Errorf("region is required to derive the %s endpoint", entry.DisplayName) + } + return fmt.Sprintf(entry.RegionEndpointFormat, strings.TrimSpace(region)), nil + } + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + parsed, err := url.Parse(endpoint) + if err != nil { + return "", fmt.Errorf("invalid endpoint %q: %w", endpoint, err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", fmt.Errorf("invalid endpoint %q: scheme must be http or https", endpoint) + } + if parsed.Host == "" { + return "", fmt.Errorf("invalid endpoint %q: host is empty", endpoint) + } + return strings.TrimSuffix(parsed.String(), "/"), nil +} diff --git a/apps/rlark/pkg/gateway/storage/provider_test.go b/apps/rlark/pkg/gateway/storage/provider_test.go new file mode 100644 index 0000000..f4741f0 --- /dev/null +++ b/apps/rlark/pkg/gateway/storage/provider_test.go @@ -0,0 +1,114 @@ +package storage + +import "testing" + +func TestLookupProviderAcceptsCanonicalDisplayNameAndAlias(t *testing.T) { + cases := []struct { + name string + input string + want string + }{ + {name: "canonical value", input: "Alibaba", want: "Alibaba"}, + {name: "case insensitive value", input: "alibaba", want: "Alibaba"}, + {name: "display name", input: "阿里云 OSS", want: "Alibaba"}, + {name: "legacy alias", input: "Aliyun OSS", want: "Alibaba"}, + {name: "surrounding spaces", input: " Minio ", want: "Minio"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + provider, ok := LookupProvider(tc.input) + if !ok { + t.Fatalf("LookupProvider(%q) reported unknown provider", tc.input) + } + if provider.Value != tc.want { + t.Fatalf("LookupProvider(%q) = %q, want %q", tc.input, provider.Value, tc.want) + } + }) + } +} + +func TestLookupProviderRejectsUnknownValue(t *testing.T) { + for _, input := range []string{"", " ", "S3 兼容", "Alibaba Cloud OSS"} { + if _, ok := LookupProvider(input); ok { + t.Fatalf("LookupProvider(%q) accepted an unknown provider", input) + } + } +} + +func TestEffectivePathStyle(t *testing.T) { + cases := []struct { + name string + provider string + requested bool + want bool + }{ + {name: "virtual hosted only forces off", provider: "Alibaba", requested: true, want: false}, + {name: "virtual hosted only by alias", provider: "Aliyun OSS", requested: true, want: false}, + {name: "generic provider keeps request", provider: "Minio", requested: true, want: true}, + {name: "generic provider keeps request off", provider: "Minio", requested: false, want: false}, + {name: "unknown provider keeps request", provider: "nonexistent", requested: true, want: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := EffectivePathStyle(tc.provider, tc.requested); got != tc.want { + t.Fatalf("EffectivePathStyle(%q, %t) = %t, want %t", tc.provider, tc.requested, got, tc.want) + } + }) + } +} + +func TestNormalizeEndpointDerivesRegionalEndpoint(t *testing.T) { + endpoint, err := NormalizeEndpoint("Alibaba", "", "cn-beijing") + if err != nil { + t.Fatalf("NormalizeEndpoint returned error: %v", err) + } + if endpoint != "https://oss-cn-beijing.aliyuncs.com" { + t.Fatalf("unexpected endpoint %q", endpoint) + } +} + +func TestNormalizeEndpointKeepsCallerEndpoint(t *testing.T) { + cases := []struct { + name string + provider string + endpoint string + want string + }{ + {name: "keeps explicit endpoint", provider: "Alibaba", endpoint: "https://oss-cn-hangzhou.aliyuncs.com", want: "https://oss-cn-hangzhou.aliyuncs.com"}, + {name: "adds https scheme", provider: "Minio", endpoint: "minio.example.com", want: "https://minio.example.com"}, + {name: "keeps http scheme", provider: "Minio", endpoint: "http://minio.example.com:9000", want: "http://minio.example.com:9000"}, + {name: "trims trailing slash", provider: "Minio", endpoint: "https://minio.example.com/", want: "https://minio.example.com"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + endpoint, err := NormalizeEndpoint(tc.provider, tc.endpoint, "cn-beijing") + if err != nil { + t.Fatalf("NormalizeEndpoint returned error: %v", err) + } + if endpoint != tc.want { + t.Fatalf("NormalizeEndpoint = %q, want %q", endpoint, tc.want) + } + }) + } +} + +func TestNormalizeEndpointRejectsIncompleteInput(t *testing.T) { + cases := []struct { + name string + provider string + endpoint string + region string + }{ + {name: "missing endpoint without regional format", provider: "Minio"}, + {name: "missing region for derived endpoint", provider: "Alibaba"}, + {name: "unsupported scheme", provider: "Minio", endpoint: "ftp://minio.example.com"}, + {name: "empty host", provider: "Minio", endpoint: "https:///bucket"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := NormalizeEndpoint(tc.provider, tc.endpoint, tc.region); err == nil { + t.Fatalf("NormalizeEndpoint(%q, %q, %q) accepted invalid input", tc.provider, tc.endpoint, tc.region) + } + }) + } +} diff --git a/apps/rlark/pkg/gateway/storage_handler.go b/apps/rlark/pkg/gateway/storage_handler.go index 5677900..8c1bb55 100644 --- a/apps/rlark/pkg/gateway/storage_handler.go +++ b/apps/rlark/pkg/gateway/storage_handler.go @@ -3,6 +3,7 @@ package gateway import ( "context" "encoding/json" + stderrors "errors" "fmt" "net/http" "regexp" @@ -11,6 +12,7 @@ import ( "strconv" "strings" "text/template" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -33,17 +35,19 @@ import ( // StorageClassData 定义StorageClass的响应数据结构. type StorageClassData struct { - Name string `json:"name"` - Clusters []string `json:"clusters"` - Description string `json:"description"` - Bucket string `json:"bucket"` - Provider string `json:"provider"` - Endpoint string `json:"endpoint"` - Region string `json:"region"` - PathStyle bool `json:"pathStyle"` - AccessKeyId string `json:"accessKeyId"` - TenantID string `json:"tenantID"` - CreatedBy string `json:"createdBy,omitempty"` + Name string `json:"name"` + StorageClassName string `json:"storageClassName"` + Clusters []string `json:"clusters"` + Description string `json:"description"` + Bucket string `json:"bucket"` + Provider string `json:"provider"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + PathStyle bool `json:"pathStyle"` + AccessKeyId string `json:"accessKeyId"` + TenantID string `json:"tenantID"` + CreatedBy string `json:"createdBy,omitempty"` + CreatedAt string `json:"createdAt"` } // Provider 定义存储提供商信息. @@ -64,7 +68,7 @@ func (g *Gateway) listStorageClass(c *gin.Context) { requestedClusters := parseStorageClassClusters(c.Query("clusters")) data := make(map[string]*StorageClassData, len(secrets)) for i := range secrets { - _, stored, err := storageClassFromSecret(&secrets[i]) + id, stored, err := storageClassFromSecret(&secrets[i]) if err != nil { log.FromContext(c.Request.Context()).Error(err, "invalid storage class replication", "secret", secrets[i].Name) respondError(c, http.StatusInternalServerError, CodeInternalError, "invalid storage class data") @@ -78,14 +82,27 @@ func (g *Gateway) listStorageClass(c *gin.Context) { } } data[stored.Name] = &StorageClassData{ - Name: stored.Name, Clusters: clusters, Description: stored.Description, Bucket: stored.Bucket, + Name: stored.Name, StorageClassName: id, Clusters: clusters, Description: stored.Description, Bucket: stored.Bucket, Provider: stored.Provider, Endpoint: stored.Endpoint, Region: stored.Region, PathStyle: stored.PathStyle, - AccessKeyId: stored.AccessKeyID, TenantID: tenant.IDOf(&secrets[i]), CreatedBy: secrets[i].Annotations[tenant.CreatedByAnnotation], + AccessKeyId: maskStorageAccessKeyID(stored.AccessKeyID), TenantID: tenant.IDOf(&secrets[i]), CreatedBy: secrets[i].Annotations[tenant.CreatedByAnnotation], CreatedAt: secrets[i].CreationTimestamp.UTC().Format(time.RFC3339), } } c.JSON(http.StatusOK, gin.H{"data": data, "success": true}) } +const storageAccessKeyMask = "*****" + +func maskStorageAccessKeyID(value string) string { + if len(value) <= 4 { + return storageAccessKeyMask + } + return value[:2] + storageAccessKeyMask + value[len(value)-2:] +} + +func isMaskedStorageAccessKeyID(value string) bool { + return strings.Contains(value, storageAccessKeyMask) +} + func parseStorageClassClusters(value string) map[string]struct{} { clusters := map[string]struct{}{} for _, cluster := range strings.Split(value, ",") { @@ -107,40 +124,13 @@ func intersectStorageClassClusters(clusters []string, requested map[string]struc return filtered } -// listProvider 列出支持的存储提供商. +// listProvider 列出支持的存储提供商. 目录由 storage.Providers 单一维护, +// 与创建/更新时的 provider 校验共用同一份数据, 避免客户端提交 rclone 无法 +// 识别的取值. func (g *Gateway) listProvider(c *gin.Context) { - providers := []Provider{ - {Name: "AWS S3", Value: "AWS"}, - {Name: "阿里云 OSS", Value: "Alibaba"}, - {Name: "腾讯云 COS", Value: "TencentCOS"}, - {Name: "华为云 OBS", Value: "HuaweiOBS"}, - {Name: "七牛云 Kodo", Value: "Qiniu"}, - {Name: "移动云 EOS", Value: "ChinaMobile"}, - {Name: "网易 NOS", Value: "Netease"}, - {Name: "UCloud US3", Value: "Ucloud"}, - {Name: "MinIO", Value: "Minio"}, - {Name: "Ceph RGW", Value: "Ceph"}, - {Name: "SeaweedFS", Value: "SeaweedFS"}, - {Name: "Cloudflare R2", Value: "Cloudflare"}, - {Name: "Google Cloud Storage", Value: "GCS"}, - {Name: "DigitalOcean Spaces", Value: "DigitalOcean"}, - {Name: "Wasabi", Value: "Wasabi"}, - {Name: "Backblaze B2 (S3)", Value: "Other"}, - {Name: "IBM COS", Value: "IBMCOS"}, - {Name: "Scaleway", Value: "Scaleway"}, - {Name: "OVHcloud", Value: "OVH"}, - {Name: "Linode / Akamai", Value: "Linode"}, - {Name: "IDrive e2", Value: "IDrive"}, - {Name: "Seagate Lyve Cloud", Value: "LyveCloud"}, - {Name: "Storj (S3 网关)", Value: "Storj"}, - {Name: "Synology C2", Value: "Synology"}, - {Name: "ArvanCloud", Value: "ArvanCloud"}, - {Name: "Hetzner", Value: "Hetzner"}, - {Name: "IONOS", Value: "IONOS"}, - {Name: "MEGA S4", Value: "Mega"}, - {Name: "RackCorp", Value: "RackCorp"}, - {Name: "Dreamhost", Value: "Dreamhost"}, - {Name: "其他 S3 兼容", Value: "Other"}, + providers := make([]Provider, 0, len(storage.Providers)) + for _, provider := range storage.Providers { + providers = append(providers, Provider{Name: provider.DisplayName, Value: provider.Value}) } c.JSON(http.StatusOK, gin.H{ @@ -156,11 +146,14 @@ const annoStorageClassDescription = "rlark.io/description" const annoStorageClassName = "rlark.io/name" const ( - storageClassVersion = "v1" - storageClassCreateAttempts = 3 - storageClassNamespacePrefix = "rlark-" - storageClassReplicationLabel = "rlark.io/storageclass-replication" - storageClassDeliveryLabel = "rlark.io/storageclass-delivery" + storageClassVersion = "v1" + storageClassCreateAttempts = 3 + storageClassValidationTimeout = 10 * time.Second + storageClassNamespacePrefix = "rlark-" + storageClassReplicationLabel = "rlark.io/storageclass-replication" + storageClassDeliveryLabel = "rlark.io/storageclass-delivery" + + CodeStorageClassRequiredFields = "STORAGE_CLASS_REQUIRED_FIELDS" ) var storageClassIDPattern = regexp.MustCompile(`^sc-[0-9a-f]{16}$`) @@ -242,7 +235,11 @@ func (g *Gateway) createStorageClass(c *gin.Context) { } stored, err := normalizeStorageClass(req) if err != nil { - respondError(c, http.StatusBadRequest, CodeInvalidRequest, err.Error()) + respondStorageClassValidationError(c, err) + return + } + if err := validateStorageClassCredentials(c.Request.Context(), stored); err != nil { + respondStorageClassCredentialValidationError(c, err) return } for range storageClassCreateAttempts { @@ -282,9 +279,35 @@ func (g *Gateway) updateStorageClass(c *gin.Context) { respondError(c, http.StatusBadRequest, CodeInvalidRequest, "storage class name cannot be changed") return } + if req.AccessKeyId == "" || isMaskedStorageAccessKeyID(req.AccessKeyId) || req.AccessKeySecret == "" { + secret, err := getStorageClassSecretByName(c.Request.Context(), g.rawClient, g.managementNamespace(), name) + if err != nil { + if errors.IsNotFound(err) { + respondError(c, http.StatusNotFound, CodeResourceNotFound, "storage class not found") + return + } + respondKubeError(c, fmt.Sprintf("get storage class replication: %v", err), err) + return + } + _, existing, err := storageClassFromSecret(secret) + if err != nil { + respondKubeError(c, fmt.Sprintf("decode storage class replication: %v", err), err) + return + } + if req.AccessKeyId == "" || isMaskedStorageAccessKeyID(req.AccessKeyId) { + req.AccessKeyId = existing.AccessKeyID + } + if req.AccessKeySecret == "" { + req.AccessKeySecret = existing.AccessKey + } + } stored, err := normalizeStorageClass(req) if err != nil { - respondError(c, http.StatusBadRequest, CodeInvalidRequest, err.Error()) + respondStorageClassValidationError(c, err) + return + } + if err := validateStorageClassCredentials(c.Request.Context(), stored); err != nil { + respondStorageClassCredentialValidationError(c, err) return } if err := g.updateStorageClassByName(c.Request.Context(), name, stored); err != nil { @@ -323,10 +346,67 @@ func (g *Gateway) deleteStorageClass(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"success": true}) } +type storageClassValidationError struct { + fields []string +} + +func (e *storageClassValidationError) Error() string { + return strings.Join(e.fields, ", ") + " are required" +} + +func requiredStorageClassFields(req CreateStorageClassRequest) []string { + var fields []string + if strings.TrimSpace(req.Name) == "" { + fields = append(fields, "name") + } + if strings.TrimSpace(req.Bucket) == "" { + fields = append(fields, "bucket") + } + if strings.TrimSpace(req.AccessKeySecret) == "" { + fields = append(fields, "access_key_secret") + } + return fields +} + +func respondStorageClassValidationError(c *gin.Context, err error) { + var validationErr *storageClassValidationError + if stderrors.As(err, &validationErr) { + respondErrorWithDetails(c, http.StatusBadRequest, CodeStorageClassRequiredFields, validationErr.Error(), map[string]any{"fields": validationErr.fields}) + return + } + respondError(c, http.StatusBadRequest, CodeInvalidRequest, err.Error()) +} + +var validateStorageClassCredentials = func(ctx context.Context, stored storedStorageClass) error { + config := storage.DefaultConfig() + config.Provider = stored.Provider + config.Endpoint = stored.Endpoint + config.AccessKeyId = stored.AccessKeyID + config.SecretAccessKey = stored.AccessKey + config.Region = stored.Region + config.UsePathStyle = stored.PathStyle + config.Bucket = stored.Bucket + client, err := storage.NewClient(config) + if err != nil { + return fmt.Errorf("create storage client: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, storageClassValidationTimeout) + defer cancel() + if err := client.ValidateBucketCredentials(ctx); err != nil { + return fmt.Errorf("head storage bucket: %w", err) + } + return nil +} + +func respondStorageClassCredentialValidationError(c *gin.Context, err error) { + log.FromContext(c.Request.Context()).Error(err, "storage credential validation failed") + respondError(c, http.StatusBadRequest, CodeInvalidRequest, "storage credentials cannot access the configured bucket") +} + func normalizeStorageClass(req CreateStorageClassRequest) (storedStorageClass, error) { name := strings.TrimSpace(req.Name) - if name == "" || req.Bucket == "" || req.AccessKeySecret == "" { - return storedStorageClass{}, fmt.Errorf("name, bucket, and access_key_secret are required") + if fields := requiredStorageClassFields(req); len(fields) != 0 { + return storedStorageClass{}, &storageClassValidationError{fields: fields} } clusters := make([]string, 0, len(req.Clusters)) for _, cluster := range req.Clusters { @@ -344,13 +424,24 @@ func normalizeStorageClass(req CreateStorageClassRequest) (storedStorageClass, e } sort.Strings(clusters) clusters = slices.Compact(clusters) - if strings.EqualFold(req.Provider, "Alibaba") { - req.Endpoint = fmt.Sprintf("%s.oss-%s.aliyuncs.com", req.Bucket, req.Region) + // Provider 必须来自已知目录: rclone 对未知 provider 只记录告警并回退到通用 + // 配置, 错误取值在数据面只表现为挂载后的 I/O 错误. 这里同时把历史客户端提交 + // 的展示名(如 "Aliyun OSS")改写为 canonical 取值. + provider, ok := storage.LookupProvider(req.Provider) + if !ok { + return storedStorageClass{}, fmt.Errorf("unsupported provider %q", req.Provider) + } + // Endpoint 只描述服务端地址, 不含 bucket: AWS SDK 与 rclone 都会自行按 + // virtual-hosted 规则拼接 bucket. + endpoint, err := storage.NormalizeEndpoint(provider.Value, req.Endpoint, req.Region) + if err != nil { + return storedStorageClass{}, err } return storedStorageClass{ Version: storageClassVersion, Name: name, Description: req.Description, Clusters: clusters, - Provider: req.Provider, Endpoint: req.Endpoint, AccessKeyID: req.AccessKeyId, AccessKey: req.AccessKeySecret, - Bucket: req.Bucket, Region: req.Region, PathStyle: req.PathStyle, + Provider: provider.Value, Endpoint: endpoint, AccessKeyID: req.AccessKeyId, AccessKey: req.AccessKeySecret, + Bucket: req.Bucket, Region: strings.TrimSpace(req.Region), + PathStyle: storage.EffectivePathStyle(provider.Value, req.PathStyle), }, nil } @@ -445,6 +536,12 @@ func storageClassFromSecret(secret *v1.Secret) (string, storedStorageClass, erro if stored.Version != storageClassVersion { return "", storedStorageClass{}, fmt.Errorf("unsupported storage class version %q", stored.Version) } + // 读取历史记录时把 provider 的展示名改写为 canonical 取值, 并按该 provider + // 实际支持的寻址方式修正 path_style, 使旧记录与新写入的记录行为一致. + if provider, ok := storage.LookupProvider(stored.Provider); ok { + stored.Provider = provider.Value + stored.PathStyle = storage.EffectivePathStyle(provider.Value, stored.PathStyle) + } return id, stored, nil } @@ -509,9 +606,8 @@ func (g *Gateway) invalidateStorageClients(name string) { } } -func (g *Gateway) getStorageClient(ctx context.Context, cluster, name string) (*storage.Client, error) { - cluster = strings.TrimPrefix(strings.TrimSpace(cluster), storageClassNamespacePrefix) - cacheKey := cluster + "/" + name +func (g *Gateway) getStorageClient(ctx context.Context, name string) (*storage.Client, error) { + cacheKey := name g.storageClientsMu.RLock() if client, ok := g.storageClients[cacheKey]; ok { g.storageClientsMu.RUnlock() @@ -527,10 +623,6 @@ func (g *Gateway) getStorageClient(ctx context.Context, cluster, name string) (* if err != nil { return nil, fmt.Errorf("decode storage class declaration: %w", err) } - if !slices.Contains(stored.Clusters, cluster) { - return nil, fmt.Errorf("storage class %s is not configured for cluster %s", name, cluster) - } - config := storage.DefaultConfig() config.Provider = stored.Provider config.Endpoint = stored.Endpoint @@ -549,13 +641,12 @@ func (g *Gateway) getStorageClient(ctx context.Context, cluster, name string) (* return client, nil } -// listStorageClassFiles handles GET /api/v1/storage/storageclass/:cluster/:name/list +// listStorageClassFiles handles GET /api/v1/storage/storageclass/:name/list // with query parameters: prefix, delimiter, marker, maxKeys. func (g *Gateway) listStorageClassFiles(c *gin.Context) { - cluster := c.Param("cluster") name := c.Param("name") - client, err := g.getStorageClient(c.Request.Context(), cluster, name) + client, err := g.getStorageClient(c.Request.Context(), name) if err != nil { respondError(c, http.StatusInternalServerError, CodeUpstreamUnavailable, err.Error()) return @@ -601,12 +692,11 @@ func (g *Gateway) listStorageClassFiles(c *gin.Context) { }) } -// uploadStorageClassFile handles POST /api/v1/storage/storageclass/:cluster/:name/upload. +// uploadStorageClassFile handles POST /api/v1/storage/storageclass/:name/upload. func (g *Gateway) uploadStorageClassFile(c *gin.Context) { - cluster := c.Param("cluster") name := c.Param("name") - client, err := g.getStorageClient(c.Request.Context(), cluster, name) + client, err := g.getStorageClient(c.Request.Context(), name) if err != nil { respondError(c, http.StatusInternalServerError, CodeUpstreamUnavailable, err.Error()) return @@ -628,10 +718,9 @@ func (g *Gateway) uploadStorageClassFile(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"key": objectKey}) } -// getStorageClassObject handles GET /api/v1/storage/storageclass/:cluster/:name/object/*key +// getStorageClassObject handles GET /api/v1/storage/storageclass/:name/object/*key // It returns a presigned URL for downloading the object. func (g *Gateway) getStorageClassObject(c *gin.Context) { - cluster := c.Param("cluster") name := c.Param("name") // gin's *key catch-all includes the leading slash; strip it so the object // key doesn't start with "/", which would produce a "//" in the presigned @@ -643,7 +732,7 @@ func (g *Gateway) getStorageClassObject(c *gin.Context) { return } - client, err := g.getStorageClient(c.Request.Context(), cluster, name) + client, err := g.getStorageClient(c.Request.Context(), name) if err != nil { respondError(c, http.StatusInternalServerError, CodeUpstreamUnavailable, err.Error()) return @@ -666,9 +755,8 @@ func (g *Gateway) getStorageClassObject(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"url": url}) } -// deleteStorageClassObject handles DELETE /api/v1/storage/storageclass/:cluster/:name/object/*key. +// deleteStorageClassObject handles DELETE /api/v1/storage/storageclass/:name/object/*key. func (g *Gateway) deleteStorageClassObject(c *gin.Context) { - cluster := c.Param("cluster") name := c.Param("name") key := strings.TrimPrefix(c.Param("key"), "/") if key == "" { @@ -676,7 +764,7 @@ func (g *Gateway) deleteStorageClassObject(c *gin.Context) { return } - client, err := g.getStorageClient(c.Request.Context(), cluster, name) + client, err := g.getStorageClient(c.Request.Context(), name) if err != nil { respondError(c, http.StatusInternalServerError, CodeUpstreamUnavailable, err.Error()) return diff --git a/apps/rlark/pkg/gateway/storage_handler_test.go b/apps/rlark/pkg/gateway/storage_handler_test.go index 5843e19..73d3e0d 100644 --- a/apps/rlark/pkg/gateway/storage_handler_test.go +++ b/apps/rlark/pkg/gateway/storage_handler_test.go @@ -4,18 +4,22 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "net/http" "testing" + "time" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/kubernetes/fake" "github.com/rlinf/rlark/apps/rlark/pkg/configs" "github.com/rlinf/rlark/apps/rlark/pkg/distribution" + "github.com/rlinf/rlark/apps/rlark/pkg/gateway/storage" ) func TestBuildStorageClassReplicationSecret(t *testing.T) { @@ -129,14 +133,377 @@ func TestListStorageClassReadsReplicationDeclarations(t *testing.T) { assert.True(t, body.Success) item, ok := body.Data["object-storage"] require.True(t, ok) + assert.Equal(t, "object-storage", item.Name) + // The human-readable name is the map key; storageClassName is the actual + // Kubernetes StorageClass name that task PVCs must reference. + assert.Equal(t, "sc-0123456789abcdef", item.StorageClassName) assert.Equal(t, []string{"rlark-cluster-b"}, item.Clusters) assert.Equal(t, "bucket", item.Bucket) assert.Equal(t, "Minio", item.Provider) + assert.Equal(t, "ac*****ss", item.AccessKeyId) + assert.Equal(t, secret.CreationTimestamp.UTC().Format(time.RFC3339), item.CreatedAt) _, err = gateway.rawClient.CoreV1().Secrets("management").Get(context.Background(), secret.Name, metav1.GetOptions{}) require.NoError(t, err) } +func TestUpdateStorageClassTreatsMaskedAccessKeyIDAsUnchanged(t *testing.T) { + gin.SetMode(gin.TestMode) + initial, err := normalizeStorageClass(CreateStorageClassRequest{ + Name: "object-storage", Clusters: []string{"rlark-cluster-a"}, Provider: "Minio", + Endpoint: "https://minio.example.com", AccessKeyId: "access", AccessKeySecret: "stored-secret", Bucket: "bucket", + }) + require.NoError(t, err) + originalValidate := validateStorageClassCredentials + var validated storedStorageClass + validateStorageClassCredentials = func(_ context.Context, stored storedStorageClass) error { + validated = stored + return nil + } + t.Cleanup(func() { validateStorageClassCredentials = originalValidate }) + secret, err := buildStorageClassReplicationSecret("management", "sc-0123456789abcdef", initial) + require.NoError(t, err) + gateway := &Gateway{ + config: Config{KubeClientConfig: configs.KubernetesClientConfig{Namespace: "management"}}, + rawClient: fake.NewSimpleClientset(secret), + storageClients: map[string]*storage.Client{}, + } + router := gin.New() + router.PUT("/storage/:name", gateway.updateStorageClass) + + response := performJSONRequest(t, router, http.MethodPut, "/storage/object-storage", map[string]any{ + "name": "object-storage", "description": "updated description", "clusters": []string{"rlark-cluster-a"}, "provider": "Minio", + "endpoint": "https://minio.example.com", "access_key_id": "ac*****ss", "access_key_secret": "", "bucket": "bucket", "path_style": true, + }) + require.Equal(t, http.StatusOK, response.Code) + + updated, err := gateway.rawClient.CoreV1().Secrets("management").Get(context.Background(), secret.Name, metav1.GetOptions{}) + require.NoError(t, err) + _, stored, err := storageClassFromSecret(updated) + require.NoError(t, err) + assert.Equal(t, "updated description", stored.Description) + assert.Equal(t, "stored-secret", stored.AccessKey) + assert.Equal(t, "access", stored.AccessKeyID) + assert.Equal(t, "stored-secret", validated.AccessKey) + assert.Equal(t, "access", validated.AccessKeyID) +} + +func TestUpdateStorageClassRetainsStoredAccessSecret(t *testing.T) { + gin.SetMode(gin.TestMode) + initial, err := normalizeStorageClass(CreateStorageClassRequest{ + Name: "object-storage", Clusters: []string{"rlark-cluster-a"}, Provider: "Minio", + Endpoint: "https://minio.example.com", AccessKeyId: "access", AccessKeySecret: "stored-secret", Bucket: "bucket", + }) + require.NoError(t, err) + originalValidate := validateStorageClassCredentials + var validated storedStorageClass + validateStorageClassCredentials = func(_ context.Context, stored storedStorageClass) error { + validated = stored + return nil + } + t.Cleanup(func() { validateStorageClassCredentials = originalValidate }) + secret, err := buildStorageClassReplicationSecret("management", "sc-0123456789abcdef", initial) + require.NoError(t, err) + gateway := &Gateway{ + config: Config{KubeClientConfig: configs.KubernetesClientConfig{Namespace: "management"}}, + rawClient: fake.NewSimpleClientset(secret), + storageClients: map[string]*storage.Client{}, + } + router := gin.New() + router.PUT("/storage/:name", gateway.updateStorageClass) + + response := performJSONRequest(t, router, http.MethodPut, "/storage/object-storage", map[string]any{ + "name": "object-storage", "clusters": []string{"rlark-cluster-a"}, "provider": "Minio", + "endpoint": "https://minio.example.com", "access_key_id": "", "access_key_secret": "", "bucket": "bucket", "path_style": true, + }) + require.Equal(t, http.StatusOK, response.Code) + + updated, err := gateway.rawClient.CoreV1().Secrets("management").Get(context.Background(), secret.Name, metav1.GetOptions{}) + require.NoError(t, err) + _, stored, err := storageClassFromSecret(updated) + require.NoError(t, err) + assert.Equal(t, "stored-secret", stored.AccessKey) + assert.Equal(t, "access", stored.AccessKeyID) + assert.True(t, stored.PathStyle) + assert.Equal(t, "stored-secret", validated.AccessKey) + assert.Equal(t, "access", validated.AccessKeyID) + assert.True(t, validated.PathStyle) +} + +func TestCreateStorageClassValidatesCredentials(t *testing.T) { + gin.SetMode(gin.TestMode) + originalValidate := validateStorageClassCredentials + var validated storedStorageClass + validateStorageClassCredentials = func(_ context.Context, stored storedStorageClass) error { + validated = stored + return nil + } + t.Cleanup(func() { validateStorageClassCredentials = originalValidate }) + + gateway := &Gateway{ + config: Config{KubeClientConfig: configs.KubernetesClientConfig{Namespace: "management"}}, + rawClient: fake.NewSimpleClientset(), + } + router := gin.New() + router.POST("/storage", gateway.createStorageClass) + + response := performJSONRequest(t, router, http.MethodPost, "/storage", map[string]any{ + "name": "object-storage", "clusters": []string{"rlark-cluster-a"}, "provider": "Minio", + "endpoint": "https://minio.example.com", "access_key_id": "access", "access_key_secret": "secret", "bucket": "bucket", "path_style": true, + }) + require.Equal(t, http.StatusOK, response.Code) + assert.Equal(t, "Minio", validated.Provider) + assert.Equal(t, "https://minio.example.com", validated.Endpoint) + assert.Equal(t, "access", validated.AccessKeyID) + assert.Equal(t, "secret", validated.AccessKey) + assert.Equal(t, "bucket", validated.Bucket) + assert.True(t, validated.PathStyle) +} + +func TestCreateStorageClassRejectsInvalidCredentials(t *testing.T) { + gin.SetMode(gin.TestMode) + originalValidate := validateStorageClassCredentials + validateStorageClassCredentials = func(context.Context, storedStorageClass) error { + return errors.New("access denied") + } + t.Cleanup(func() { validateStorageClassCredentials = originalValidate }) + + gateway := &Gateway{ + config: Config{KubeClientConfig: configs.KubernetesClientConfig{Namespace: "management"}}, + rawClient: fake.NewSimpleClientset(), + } + router := gin.New() + router.POST("/storage", gateway.createStorageClass) + + response := performJSONRequest(t, router, http.MethodPost, "/storage", map[string]any{ + "name": "object-storage", "clusters": []string{"rlark-cluster-a"}, "provider": "Minio", + "endpoint": "https://minio.example.com", "access_key_secret": "secret", "bucket": "bucket", + }) + require.Equal(t, http.StatusBadRequest, response.Code) + var body ErrorResponse + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + assert.Equal(t, CodeInvalidRequest, body.Code) + assert.Equal(t, "storage credentials cannot access the configured bucket", body.Message) +} + +func TestUpdateStorageClassRejectsInvalidCredentials(t *testing.T) { + gin.SetMode(gin.TestMode) + initial, err := normalizeStorageClass(CreateStorageClassRequest{ + Name: "object-storage", Clusters: []string{"rlark-cluster-a"}, Provider: "Minio", + Endpoint: "https://minio.example.com", AccessKeyId: "access", AccessKeySecret: "stored-secret", Bucket: "bucket", + }) + require.NoError(t, err) + secret, err := buildStorageClassReplicationSecret("management", "sc-0123456789abcdef", initial) + require.NoError(t, err) + originalValidate := validateStorageClassCredentials + validateStorageClassCredentials = func(context.Context, storedStorageClass) error { + return errors.New("access denied") + } + t.Cleanup(func() { validateStorageClassCredentials = originalValidate }) + + gateway := &Gateway{ + config: Config{KubeClientConfig: configs.KubernetesClientConfig{Namespace: "management"}}, + rawClient: fake.NewSimpleClientset(secret), + storageClients: map[string]*storage.Client{}, + } + router := gin.New() + router.PUT("/storage/:name", gateway.updateStorageClass) + + response := performJSONRequest(t, router, http.MethodPut, "/storage/object-storage", map[string]any{ + "name": "object-storage", "clusters": []string{"rlark-cluster-a"}, "provider": "Minio", + "endpoint": "https://minio.example.com", "access_key_id": "", "access_key_secret": "", "bucket": "bucket", + }) + require.Equal(t, http.StatusBadRequest, response.Code) + var body ErrorResponse + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + assert.Equal(t, CodeInvalidRequest, body.Code) + assert.Equal(t, "storage credentials cannot access the configured bucket", body.Message) + + current, err := gateway.rawClient.CoreV1().Secrets("management").Get(context.Background(), secret.Name, metav1.GetOptions{}) + require.NoError(t, err) + _, stored, err := storageClassFromSecret(current) + require.NoError(t, err) + assert.Equal(t, initial, stored) +} + +func TestUpdateStorageClassReportsStructuredRequiredFieldErrors(t *testing.T) { + gin.SetMode(gin.TestMode) + initial, err := normalizeStorageClass(CreateStorageClassRequest{ + Name: "object-storage", Clusters: []string{"rlark-cluster-a"}, Provider: "Minio", + Endpoint: "https://minio.example.com", AccessKeyId: "access", AccessKeySecret: "stored-secret", Bucket: "bucket", + }) + require.NoError(t, err) + secret, err := buildStorageClassReplicationSecret("management", "sc-0123456789abcdef", initial) + require.NoError(t, err) + gateway := &Gateway{ + config: Config{KubeClientConfig: configs.KubernetesClientConfig{Namespace: "management"}}, + rawClient: fake.NewSimpleClientset(secret), + storageClients: map[string]*storage.Client{}, + } + router := gin.New() + router.PUT("/storage/:name", gateway.updateStorageClass) + + response := performJSONRequest(t, router, http.MethodPut, "/storage/object-storage", map[string]any{ + "name": "object-storage", "clusters": []string{"rlark-cluster-a"}, "provider": "Minio", + "endpoint": "https://minio.example.com", "bucket": "", + }) + require.Equal(t, http.StatusBadRequest, response.Code) + var body ErrorResponse + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + assert.Equal(t, CodeStorageClassRequiredFields, body.Code) + assert.Equal(t, []any{"bucket"}, body.Details["fields"]) +} + +func TestNormalizeStorageClassResolvesProviderAndEndpoint(t *testing.T) { + cases := []struct { + name string + req CreateStorageClassRequest + wantProvider string + wantEndpoint string + wantPathStyle bool + }{ + { + name: "derives the regional endpoint when omitted", + req: CreateStorageClassRequest{ + Name: "oss", Clusters: []string{"rlark-cluster-a"}, Provider: "Alibaba", + Region: "cn-beijing", AccessKeySecret: "secret", Bucket: "bucket", + }, + wantProvider: "Alibaba", wantEndpoint: "https://oss-cn-beijing.aliyuncs.com", + }, + { + name: "keeps the caller endpoint without embedding the bucket", + req: CreateStorageClassRequest{ + Name: "oss", Clusters: []string{"rlark-cluster-a"}, Provider: "Alibaba", + Endpoint: "https://oss-cn-beijing.aliyuncs.com", Region: "cn-beijing", + AccessKeySecret: "secret", Bucket: "bucket", + }, + wantProvider: "Alibaba", wantEndpoint: "https://oss-cn-beijing.aliyuncs.com", + }, + { + name: "rewrites a legacy display name to the canonical value", + req: CreateStorageClassRequest{ + Name: "oss", Clusters: []string{"rlark-cluster-a"}, Provider: "Aliyun OSS", + Region: "cn-beijing", AccessKeySecret: "secret", Bucket: "bucket", + }, + wantProvider: "Alibaba", wantEndpoint: "https://oss-cn-beijing.aliyuncs.com", + }, + { + name: "forces virtual-hosted addressing for providers that reject path style", + req: CreateStorageClassRequest{ + Name: "oss", Clusters: []string{"rlark-cluster-a"}, Provider: "Alibaba", + Region: "cn-beijing", PathStyle: true, AccessKeySecret: "secret", Bucket: "bucket", + }, + wantProvider: "Alibaba", wantEndpoint: "https://oss-cn-beijing.aliyuncs.com", + }, + { + name: "keeps path style for generic S3 backends", + req: CreateStorageClassRequest{ + Name: "minio", Clusters: []string{"rlark-cluster-a"}, Provider: "Minio", + Endpoint: "minio.example.com", PathStyle: true, AccessKeySecret: "secret", Bucket: "bucket", + }, + wantProvider: "Minio", wantEndpoint: "https://minio.example.com", wantPathStyle: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stored, err := normalizeStorageClass(tc.req) + require.NoError(t, err) + assert.Equal(t, tc.wantProvider, stored.Provider) + assert.Equal(t, tc.wantEndpoint, stored.Endpoint) + assert.Equal(t, tc.wantPathStyle, stored.PathStyle) + assert.NotContains(t, stored.Endpoint, tc.req.Bucket+".") + }) + } +} + +func TestNormalizeStorageClassRejectsInvalidProviderAndEndpoint(t *testing.T) { + cases := []struct { + name string + req CreateStorageClassRequest + }{ + { + name: "unknown provider", + req: CreateStorageClassRequest{ + Name: "oss", Clusters: []string{"rlark-cluster-a"}, Provider: "Alibaba Cloud OSS", + Endpoint: "https://oss-cn-beijing.aliyuncs.com", AccessKeySecret: "secret", Bucket: "bucket", + }, + }, + { + name: "empty provider", + req: CreateStorageClassRequest{ + Name: "oss", Clusters: []string{"rlark-cluster-a"}, + Endpoint: "https://oss-cn-beijing.aliyuncs.com", AccessKeySecret: "secret", Bucket: "bucket", + }, + }, + { + name: "missing endpoint and region", + req: CreateStorageClassRequest{ + Name: "oss", Clusters: []string{"rlark-cluster-a"}, Provider: "Alibaba", + AccessKeySecret: "secret", Bucket: "bucket", + }, + }, + { + name: "missing endpoint for a generic backend", + req: CreateStorageClassRequest{ + Name: "minio", Clusters: []string{"rlark-cluster-a"}, Provider: "Minio", + AccessKeySecret: "secret", Bucket: "bucket", + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := normalizeStorageClass(tc.req) + require.Error(t, err) + }) + } +} + +func TestGetStorageClientDoesNotRequireConfiguredCluster(t *testing.T) { + stored, err := normalizeStorageClass(CreateStorageClassRequest{ + Name: "object-storage", Clusters: []string{"rlark-cluster-a"}, Provider: "Minio", + Endpoint: "https://minio.example.com", AccessKeyId: "access", AccessKeySecret: "secret", Bucket: "bucket", + }) + require.NoError(t, err) + secret, err := buildStorageClassReplicationSecret("management", "sc-0123456789abcdef", stored) + require.NoError(t, err) + gateway := &Gateway{ + config: Config{KubeClientConfig: configs.KubernetesClientConfig{Namespace: "management"}}, + rawClient: fake.NewSimpleClientset(secret), + storageClients: map[string]*storage.Client{}, + } + + client, err := gateway.getStorageClient(context.Background(), "object-storage") + require.NoError(t, err) + assert.NotNil(t, client) +} + +func TestStorageClassFromSecretCanonicalizesLegacyRecords(t *testing.T) { + // 旧版 UI 提交的是展示名, 并且允许把 path_style 置为 true, 这里验证读取 + // 路径会把两者修正为与新写入记录一致的形式. + stored := storedStorageClass{ + Version: storageClassVersion, Name: "object-storage", Clusters: []string{"rlark-cluster-a"}, + Provider: "Aliyun OSS", Endpoint: "https://oss-cn-beijing.aliyuncs.com", Region: "cn-beijing", + Bucket: "bucket", PathStyle: true, + } + storedJSON, err := json.Marshal(stored) + require.NoError(t, err) + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: storageClassReplicationName("sc-0123456789abcdef"), + Namespace: "management", + Labels: map[string]string{storageClassReplicationLabel: "true"}, + }, + Type: distribution.ReplicationSecretType, + Data: map[string][]byte{"storageclass": storedJSON}, + } + + id, decoded, err := storageClassFromSecret(secret) + require.NoError(t, err) + assert.Equal(t, "sc-0123456789abcdef", id) + assert.Equal(t, "Alibaba", decoded.Provider) + assert.False(t, decoded.PathStyle) +} + func nestedData(t *testing.T, object *unstructured.Unstructured) map[string]string { t.Helper() data, found, err := unstructured.NestedStringMap(object.Object, "data") diff --git a/apps/rlark/pkg/gateway/swagger_test.go b/apps/rlark/pkg/gateway/swagger_test.go index 4bdf390..0e9f9f2 100644 --- a/apps/rlark/pkg/gateway/swagger_test.go +++ b/apps/rlark/pkg/gateway/swagger_test.go @@ -152,10 +152,10 @@ var routesUndocumentedInSwagger = map[string]struct{}{ "PUT /api/v1/storage/storageclass/{name}": {}, "DELETE /api/v1/storage/storageclass/{name}": {}, "GET /api/v1/storage/storageclass/provider": {}, - "GET /api/v1/storage/storageclass/{name}/{cluster}/list": {}, - "POST /api/v1/storage/storageclass/{name}/{cluster}/upload": {}, - "GET /api/v1/storage/storageclass/{name}/{cluster}/object/{key}": {}, - "DELETE /api/v1/storage/storageclass/{name}/{cluster}/object/{key}": {}, + "GET /api/v1/storage/storageclass/{name}/list": {}, + "POST /api/v1/storage/storageclass/{name}/upload": {}, + "GET /api/v1/storage/storageclass/{name}/object/{key}": {}, + "DELETE /api/v1/storage/storageclass/{name}/object/{key}": {}, "GET /api/v1/addons": {}, "GET /api/v1/addons/{name}": {}, "GET /api/v1/installed-addons": {}, diff --git a/apps/rlark/pkg/rlarktools/raylauncher/launcher.go b/apps/rlark/pkg/rlarktools/raylauncher/launcher.go index 56169b0..2c21607 100644 --- a/apps/rlark/pkg/rlarktools/raylauncher/launcher.go +++ b/apps/rlark/pkg/rlarktools/raylauncher/launcher.go @@ -178,9 +178,8 @@ func runHead(ctx context.Context, config Config) error { if err := commandRunner(ctx, config.BashBinary, "-c", script); err != nil { return preserveExit("run script", err) } - return nil } - <-ctx.Done() + <-ctx.Done() // 执行完 RLARK_RUN_SCRIPT 开始阻塞，保证 pod 不退出 return ctx.Err() } diff --git a/test/kindtest/.gitignore b/test/kindtest/.gitignore new file mode 100644 index 0000000..8e38d01 --- /dev/null +++ b/test/kindtest/.gitignore @@ -0,0 +1,3 @@ +# 本地运行绑定与产物不入库；模板见 config.example.yaml +config.yaml +reports/ diff --git a/test/kindtest/README.md b/test/kindtest/README.md new file mode 100644 index 0000000..a99b42d --- /dev/null +++ b/test/kindtest/README.md @@ -0,0 +1,357 @@ +# kindtest — 代码优先的 Kind 回归框架 + +**配置只描述“在哪里测”**（集群、kubeconfig、镜像、命名版本、报告目录）， +**“测什么”一律用 Python 代码写**。框架只提供很少的语法糖，其余复用 Python +自身的特性（装饰器、生成器、参数注入、异常）。 + +报告采用分层语义： + +``` +测试流程 (flow) → 测试观测点 (observation point) → 测试项 (item = 报告的一行) +``` + +* **测试流程**：用装饰器定义的一段过程，只运行一次；它可以建立/销毁环境。 +* **测试观测点**：流程里的一个 `with t.observe("REG-01"):` 块，评估一个或多个测试项。 +* **测试项**：报告矩阵里的行；同一个 id 可以在同一流程的多个观测点出现，结果合并。 + id 是矩阵的主键，**不同流程/用例不能重复声明同一个 id**——框架启动时直接报错，不会让后者静默覆盖前者。 + +--- + +## 目录 + +``` +test/kindtest/ + kindtest/ # 框架包（无第三方依赖，仅 PyYAML） + registry.py # group / flow / item / case / fixture / 参数展开 + context.py # 上下文 t：kubectl 操作、渲染、等待、证据、断言、观测点 + kube.py # Kube / Cluster / Kind 句柄 + versions.py # 命名版本与滚动部署（config.versions / t.upgrade） + runner.py # 全表先行、流程执行、实时报告 + report.py / model.py / config.py / shell.py / cli.py + kt # 启动器（纯脚本，无需安装） + config.example.yaml # 配置模板（复制为 config.yaml 后填写） + config.yaml # 本地配置：不入库（见 .gitignore） + tests/ # 框架自带样例 + 00_env.py # ENV-*：环境健康（单点 case） + 05_mig.py # MIG-*：命名版本在途升级（未配置 versions 则“受限”） + 10_ray.py # REG-*：跨集群 Ray（一个流程 + 四个观测点） + 20_task.py # TASK-*：单任务生命周期（一个流程 + 三个观测点） + 90_cleanup.py # CLEAN-*：清理与最终状态 + test_framework.py # 框架自检（无需集群）：python3 test_framework.py + reports/ # 运行产物（.gitignore 忽略） +``` + +--- + +## 框架与测例分离 + +框架和测例放两个地方；**测例目录里只放自己的 `config.yaml` 和测例代码**， +运行只用框架自带的启动器（第 ① 种，纯脚本，无需安装）： + +```bash +cd /your/own/tests # 你自己写测例的目录 +KT=/test/kindtest/kt # 指向本仓库框架的启动器（绝对或相对路径） +$KT validate -c config.yaml # 校验配置 +$KT plan -c config.yaml # 列出流程/观测点 +$KT run -c config.yaml # 实机执行，实时写报告 +$KT run -c config.yaml --only REG # 只跑某组（组名/流程名精确匹配） +$KT run -c config.yaml --only REG-01 # 只跑某项（id 精确匹配） +$KT run -c config.yaml --only 'REG-0*' # 尾部 * 通配，按前缀选多项 +$KT report -c config.yaml # 由 manifest 重新生成报告 +``` + +测例代码**只 import `kindtest`**，不引用框架目录里的任何文件。`config.yaml` 里 +`tests.dir` / `report.dir` / `manifests.dir` 相对配置文件解析，也接受绝对路径。 + +框架自带样例的用法： + +```bash +cd /test/kindtest +cp config.example.yaml config.yaml # 改成你自己的运行环境，config.yaml 不入库 +./kt validate -c config.yaml +./kt plan -c config.yaml +./kt run -c config.yaml +``` + +--- + +## 写测例 + +### 一个流程 + 多个观测点 + +```python +from kindtest import group + +reg = group("REG", title="基本功能回归") + +# ① 装饰器定义流程，items= 先声明本次流程的所有观测点（报告全表） +@reg.flow("跨集群 StatefulSet Ray", items=[ + ("REG-01", "Ray Active", "head+2 worker，3 Active"), + ("REG-02", "rank 和放置", "head 与 worker 落在配置的节点"), +]) +def cross_cluster_ray(t): + job = f"{t.run_id}-ray" + t.delete_job(job) + t.apply(t.render(RAY_JOB), on="mgmt") + try: + t.require(lambda: t.kexists("pod", t.head_pod(job), on="data", cluster="a"), + timeout=300, desc="head 出现") + + # ② 观测点：每个 with 产出一行报告 + with t.observe("REG-01"): + t.evidence(t.kexec("a", t.head_pod(job), "ray", "status").combined(), "REG-ray.txt") + t.ge(t.ray_active("a", t.head_pod(job)), 3, "Ray Active") + with t.observe("REG-02"): + t.eq(t.pod_node("a", t.head_pod(job)), t.config.get("placement.a"), "head 节点") + finally: + t.delete_job(job) +``` + +语义： + +* 流程启动时，`items=` 声明的测试项先建成「未执行」的全表； +* 每退出一个 `observe`，实时刷新该行（断言累加成「实际结果」）； +* 流程中途抛 `Blocked`，已声明但未观测到的项统一标 `Blocked`； +* 同一 id 可在多个观测点出现，结果合并。 + +也可以用叠加的 `@group.item(...)` 声明观测点： + +```python +@reg.flow("跨集群 StatefulSet Ray") +@reg.item("REG-01", "Ray Active", expect="3 Active") +@reg.item("REG-02", "rank 和放置", expect="head 与 worker 落在配置的节点") +def cross_cluster_ray(t): + ... +``` + +### 单点 case（一个观测点的流程） + +```python +env = group("ENV", title="环境和组件健康") + +@env.case("ENV-01", "Kind 节点容器", expect="配置中的集群节点容器全部运行", source="环境") +def kind_containers(t): + names = [str(t.config.get("clusters.control.name"))] + [t.cl(k).name for k in t.data_keys()] + expected = [n for name in names for n in t.kind.nodes(name)] + up = set(t.kind.containers_up()) + return t.eq([n for n in expected if n not in up], [], "未运行节点容器") # 返回值即“实际结果” +``` + +规则： + +1. 第一个参数永远是上下文 `t`，其余参数按名字从 **参数 / fixture** 注入。 +2. 想记录「受限 / 前置不满足」，抛 `Restricted` / `Blocked`，或用 `t.restrict(...)` / `t.block(...)`。 +3. 断言即事实：`t.eq/ge/...` 失败抛 `AssertionError`，成功返回可审计的事实串。 + +### 参数化：一行生成多行 + +```python +@env.case("ENV-02", "Kubernetes 节点 Ready", expect="各集群节点全部 Ready", source="环境", + params=[{"cluster": "control"}, {"cluster": "a"}, {"cluster": "b"}], + ids=["control", "data-a", "data-b"]) +def nodes_ready(t, cluster): + kube = t.control if cluster == "control" else t.data(cluster) + lines = [ln for ln in kube("get", "nodes", "--no-headers").stdout.splitlines() if ln.strip()] + return t.eq(len([l for l in lines if " Ready" in l]), len(lines), f"{cluster} Ready 节点") +``` + +生成 `ENV-02[control]`、`ENV-02[data-a]`、`ENV-02[data-b]`。 + +### 共享前置：生成器 fixture + +```python +from kindtest import fixture + +@fixture(scope="group") # "group"（一组共享）或 "case"（每次执行独立） +def ray(t): + t.delete_job("{{runId}}-ray") + t.apply(t.render(RAY_JOB), on="mgmt") + t.require(lambda: t.kexists("pod", head, on="data", cluster="a"), desc="head 出现") + yield {"head": head, "workers": [...]} # yield 之前是 setup + t.delete_job("{{runId}}-ray") # yield 之后是 teardown，永远执行 +``` + +--- + +## 基本操作函数（挂在 `t` 上） + +| 类别 | 函数 | +| --- | --- | +| 通用 kubectl | `t.kget / t.kjp / t.kjson / t.knames / t.kexists / t.kexec / t.klogs / t.kwait_pod` | +| 变更 | `t.apply / t.apply_manifest / t.kdelete / t.kpatch` | +| Job/Task | `t.delete_job / t.job_stopped / t.restart_job / t.task_name / t.task_uid / t.task_phase` | +| Pod/网络 | `t.head_pod / t.pod_ready / t.pod_node / t.dns_resolve / t.tcp_connect / t.ray_active` | +| Agent | `t.agent_image / t.agent_ready / t.set_agent_image / t.rollout_agent` | +| 版本（见下节） | `t.version_names / t.version_image / t.version_ref / t.current_version / t.upgrade(name) / t.deploy_version(name)` | +| 等待 | `t.wait / t.require / t.wait_pod / t.sleep` | +| 断言（累加进观测点） | `t.eq / t.ne / t.ge / t.le / t.truthy / t.contains / t.matches / t.succeeded / t.failed / t.nonempty` | +| 判定与证据 | `t.restrict / t.block / t.skip / t.evidence` | +| 句柄与模板 | `t.mgmt / t.control / t.data("a") / t.cl("a") / t.kind / t.render(text) / t.manifest_text(name) / t.vars` | + +`on=` 取值 `mgmt`（KCP）/`control`/`data`；`data` 时带 `cluster="a"|"b"`；`ns` 省略即用各侧默认命名空间。 +`Kube(...)` 直接调用是**原样** `kubectl`（不加默认命名空间）；需要默认命名空间时用上面的方法。 +句柄自身的方法（`get/jp/json/exists/names/delete/apply/patch/exec/logs/rollout_status` 等）见 `kindtest/kube.py`；`t.kind` 提供宿主机 `kind`/`docker` 能力（节点容器、`image_id`、`load`）。 + +--- + +## 配置 `config.yaml` + +模板见 `config.example.yaml`：复制为 `config.yaml`（不入库）后填你自己的环境。 +只放环境事实。`tests.dir` / `report.dir` / `manifests.dir` 相对配置文件解析，也接受绝对路径。 + +```yaml +run: + id: kt-1023 # 对象名前缀 + rlark.io/test-run 标签值 + label: rlark.io/test-run + domain: default + theme: kindtest + +clusters: + control: + name: control + kubeconfig: /path/to/control/kubeconfig + kcp: /path/to/control/kcp.kubeconfig + namespace: rlark-system + data: + a: {name: worker-a, clusterId: rlark-data-a, kubeconfig: /path/to/worker-a/kubeconfig} + b: {name: worker-b, clusterId: rlark-data-b, kubeconfig: /path/to/worker-b/kubeconfig} + +versions: # 可选：多版本与升级，见“多版本与升级” + initial: target + definitions: + base: {ref: main@, images: {rlark: rlark:baseline, ui: rlark-ui:baseline}} + target: {ref: main@, images: {rlark: rlark:target, ui: rlark-ui:target}} + rollout: + data: + - {kind: deploy, name: rlark-agent, container: rlark-agent, slot: rlark, args: ["--image="]} + - {kind: ds, name: rlark-agent-node, container: rlark-agent-node, slot: rlark} + +images: + task: task:test + +report: + dir: reports + +tests: + dir: tests +``` + +--- + +## 前置检查（preflight） + +`kt run` 在任何流程开始之前先做前置检查，并在需要时先部署 `versions.initial`： + +- PATH 中是否存在 `kubectl`； +- 每个 `Kube`（KCP、控制集群、各数据集群）的 kubeconfig 是否可读； +- 各集群是否可达（`kubectl get --raw=/readyz`）。 + +任一项失败、或初始版本部署失败，框架都会把**整个矩阵标记为 `Blocked`**（不是失败）， +并在报告**顶层**用醒目横幅写明原因与修复提示。这样“环境根本没起来”不会被误读成 +一堆断言失败或一份看似正常的报告。区块失败与 `Blocked` 都不计入通过。 + +--- + +## 多版本与升级 + +配置里可以定义**多个命名版本**（每个是 `{ref, image}`），并声明切换版本时要 +滚动的工作负载。框架会在**任何流程开始之前**先部署 `versions.initial`；测例里 +再按需调用 `t.upgrade(代号)` 在受控时点切到某个已定义版本。 + +```yaml +versions: + initial: target # 运行开始先部署的版本（代号） + definitions: # 一个版本 = ref + 若干命名镜像（不同组件不同镜像） + base: + ref: main@aaaaaaa + images: {rlark: rlark:baseline-aaaaaaa, ui: rlark-ui:baseline} + target: + ref: main@bbbbbbb + images: {rlark: rlark:target-bbbbbbb, ui: rlark-ui:target} + rollout: # 切换版本时滚动哪些负载，以及各自用哪个镜像槽 + control: # 控制集群内 + - {kind: deploy, name: rlark-controller-manager, container: rlark-controller-manager, slot: rlark} + - {kind: deploy, name: rlark-gateway, container: rlark-gateway, slot: rlark} + - {kind: deploy, name: rlark-server, container: rlark-server, slot: rlark} + - {kind: deploy, name: rlark-ui, container: rlark-ui, slot: ui} + data: # 每个数据集群内 + - {kind: deploy, name: rlark-agent, container: rlark-agent, slot: rlark, args: ["--image="]} + - {kind: ds, name: rlark-agent-node, container: rlark-agent-node, slot: rlark} +``` + +规则： + +* 一个版本可含**多个命名镜像**（`images: {rlark: …, ui: …}`）；`rollout` 的每个 + 目标用 `slot:` 选择用哪个镜像。省略 `slot` 时只用 `default` 槽或唯一镜像； + 定义了多个镜像槽却没有 `default` 的版本**必须显式指定 `slot`**，否则报错，不会 + 静默挑一个。单镜像版本也可写成 `image: `（等价于 `default` 槽）。UI 与 + rlark 是不同镜像，因此必须分开成两个槽。 +* `rollout.control` 作用于控制集群，`rollout.data` 作用于**每个**数据集群； + `kind` 取 `deploy` / `ds` / `sts`。切换时把该容器的镜像与 `args` 合并成**一次 + patch**（一次 rollout，无“新镜像 + 旧参数”的中间态），再等待 `rollout status`。 +* 目标的 `args` 列出要改写的参数前缀（如 `--image=`）。框架只改写**已存在**的 + 前缀；前缀缺失时默认报错（避免悄悄给容器追加参数、让它跑在与环境不同的配置上）， + 确需缺失即追加时在该目标上设置 `appendArgs: true`。 +* **幂等**：live 镜像已等于目标版本时不做改动，只校验就绪；重复调用或部署 + 当前版本都很便宜。 +* `versions.rollout` 缺省时框架**不实际切换**，只在报告登记版本声明； + `t.upgrade()` 此时抛 `Restricted`。未配置 `versions` 时一切照旧， + `t.version_names()` 返回空列表。 +* 配置 `versions` 后，报告的“目标版本/目标镜像”取 `initial` 的 `ref`/`image`， + “基线”取 `definitions` 中名为 `base`/`baseline` 的版本（若有）。**取值**接口 + （`config.target_image`/`baseline_image`、`t.version_image(代号, 槽)`、 + `{{targetImage}}`/`{{baselineImage}}`）按严格规则解析：用 `default` 槽或唯一 + 镜像；同一版本多槽且无 `default` 时必须指定槽，否则抛 `VersionError`，不会 + 静默挑一个。**展示**接口（报告“镜像”列）则列出全部槽（`rlark=…、ui=…`）。 + 模板里用 `{{image.<代号>.<槽>}}` 引用具体镜像；`{{image.<代号>}}` 在唯一/default + 时给出该镜像、多槽时列出全部槽；`{{targetImage}}`/`{{baselineImage}}` 只在模板 + 真正用到时解析，未用到的模板不会因目标镜像不明确而报错。 + +测例写法： + +```python +from kindtest import Restricted, group + +mig = group("MIG", title="在途升级和不变量") + +@mig.flow("命名版本在途升级", items=[ + ("MIG-01", "初始版本就绪", "运行开始部署的 initial 可被识别"), + ("MIG-02", "升级到目标版本", "t.upgrade(目标) 后镜像切换且 Ready"), +]) +def named_version_upgrade(t): + if not t.versions.enabled or not t.versions.rollout_configured: + raise Restricted("未配置 versions.rollout") # 整组记为“受限” + with t.observe("MIG-01"): + t.eq(t.current_version(), t.versions.initial, "当前版本") + with t.observe("MIG-02"): + t.upgrade("target") # 也可写 t.deploy_version("target") + t.eq(t.current_version(), "target", "升级后版本") +``` + +`t.current_version()` 通过比对 live 目标镜像与 `definitions` 得出；`t.upgrade(name)` +对未知代号抛 `VersionError`，rollout 超时同样抛错（观测点内记为失败，观测点外 +中止该流程并把其余项标 `Blocked`）。框架自带样例见 `tests/05_mig.py`。 + +--- + +## 报告与证据 + +``` +/ + full-test-result.md # 实时更新的矩阵报告 + run-manifest.json # 机器可读结果 + run-.log # 每条命令与抛错 + evidence/ # t.evidence 追加的证据 +``` + +矩阵类别顺序为 `ENV → BUILD → UNIT → BIND → MIG → CRD → REG → TASK → SSH → VALID → OBS → CLEAN`； +目标 ref 变化时，上一轮报告、清单与证据自动存档为 `full-test-result-.md`、 +`run-manifest-.json`、`evidence-/`。 + +--- + +## 约定 + +- 每轮 Job/Task 用唯一 `run.id` 前缀与配置的标签（默认 `rlark.io/test-run`），只清理本轮拥有的对象。 +- 不触碰与本轮无关的既有对象。 +- 只观察与报告，不为让测试通过而改动产品代码。 diff --git a/test/kindtest/config.example.yaml b/test/kindtest/config.example.yaml new file mode 100644 index 0000000..a7c0ed8 --- /dev/null +++ b/test/kindtest/config.example.yaml @@ -0,0 +1,64 @@ +# --------------------------------------------------------------------------- +# kindtest 运行配置【模板】。 +# +# 复制为 config.yaml 后，改成你自己的运行环境；config.yaml 不入库（见 .gitignore）。 +# 本文件只放“环境事实”，“测什么”一律写进 tests/*.py。 +# tests.dir / report.dir / manifests.dir 相对本文件解析，也接受绝对路径。 +# --------------------------------------------------------------------------- + +run: + id: kt-0001 # 对象名前缀 + rlark.io/test-run 标签值，每轮唯一 + label: rlark.io/test-run + domain: default + theme: kindtest + title: kindtest 回归结果 + +clusters: + control: + name: control # 控制集群名（报告与节点查询用） + kubeconfig: /path/to/control/kubeconfig + kcp: /path/to/control/kcp.kubeconfig + namespace: rlark-system + data: + a: + name: worker-a # 数据集群名 + clusterId: rlark-data-a # 管理面命名空间 + kubeconfig: /path/to/worker-a/kubeconfig + b: + name: worker-b + clusterId: rlark-data-b + kubeconfig: /path/to/worker-b/kubeconfig + +versions: # 多版本 / 在途升级：报告基线/目标也从这里派生 + initial: target # 运行开始时框架先部署这个版本（definitions 里的代号） + definitions: # 代号 -> {git ref, 命名镜像}；测例用 t.upgrade(代号) 切换 + base: # 名为 base/baseline 的版本用作报告的“基线” + ref: + images: {rlark: rlark:baseline, ui: rlark-ui:baseline} + target: # 名为 target（或 initial）的版本用作报告的“目标” + ref: + images: {rlark: rlark:target, ui: rlark-ui:target} + rollout: # 切换版本时滚动的负载；缺省则只声明版本、不实际切换 + control: # 控制集群（rlark-control）内的负载 + - {kind: deploy, name: rlark-controller-manager, container: rlark-controller-manager, slot: rlark} + - {kind: deploy, name: rlark-gateway, container: rlark-gateway, slot: rlark} + - {kind: deploy, name: rlark-server, container: rlark-server, slot: rlark} + - {kind: deploy, name: rlark-ui, container: rlark-ui, slot: ui} + data: # 每个数据集群都要滚动的负载 + - {kind: deploy, name: rlark-agent, container: rlark-agent, slot: rlark, args: ["--image="]} + - {kind: ds, name: rlark-agent-node, container: rlark-agent-node, slot: rlark} + +images: + task: task:test # 场景工作负载镜像 + base: [] # 可选：离线基础镜像预检列表（ENV-07 使用） + +placement: # 可选：样例清单的节点选择器（{{nodeA}}/{{nodeB}}/{{nodeB2}}） + a: + b: + b2: + +report: + dir: reports # 相对本文件；也可写绝对路径 + +tests: + dir: tests # 相对本文件；也可写绝对路径 diff --git a/test/kindtest/kindtest/__init__.py b/test/kindtest/kindtest/__init__.py new file mode 100644 index 0000000..5b833a4 --- /dev/null +++ b/test/kindtest/kindtest/__init__.py @@ -0,0 +1,88 @@ +"""kindtest - a code-first Kind regression framework for RLark. + +Configuration holds only environment facts (clusters, images, named versions, +report location). Tests are ordinary Python functions and follow the reporting +model + + 测试流程 (flow) -> 测试观测点 (observation point) -> 测试项 (report item) + +Quick start:: + + from kindtest import group + + reg = group("REG", title="基本功能回归") + + @reg.flow("跨集群 Ray", items=[ + ("REG-01", "Ray Active", "3 Active"), + ("REG-02", "rank 和放置", "head 与 worker 落在配置的节点"), + ]) + def cross_cluster_ray(t): + job = f"{t.run_id}-ray" + t.delete_job(job) + t.apply(t.render(RAY_JOB), on="mgmt") + try: + t.require(lambda: t.kexists("pod", t.head_pod(job), on="data", cluster="a"), + timeout=300, desc="head 出现") + with t.observe("REG-01"): + t.ge(t.ray_active("a", t.head_pod(job)), 3, "Ray Active") + with t.observe("REG-02"): + t.eq(t.pod_node("a", t.head_pod(job)), t.config.get("placement.a"), "head 节点") + finally: + t.delete_job(job) + +A single-observation case is just sugar:: + + @env.case("ENV-01", "Kind 节点容器", expect="全部运行") + def kind_containers(t): + return f"{len(t.kind.containers_up())} 个容器 Up" + +See ``README.md`` and the ``tests/`` directory. +""" + +from __future__ import annotations + +from .context import Context +from .kube import Cluster, Kind, Kube +from .model import Case, RunReport, Status +from .registry import Blocked, Restricted, fixture, group +from .versions import VersionError + +__all__ = [ + "__version__", + "group", + "case", + "flow", + "fixture", + "Blocked", + "Restricted", + "VersionError", + "Context", + "Cluster", + "Kube", + "Kind", + "Case", + "RunReport", + "Status", +] + +__version__ = "0.2.0" + + +def _default_group(kwargs: dict): + from .registry import group as _group + + return _group(kwargs.pop("group", "MISC"), title=kwargs.pop("title", None)) + + +def case(*args, **kwargs): + """Decorator ``@case(...)`` using the default group. + + Prefer ``@mygroup.case(...)``. This helper exists so a single-category test + file can avoid creating an explicit group object. + """ + return _default_group(kwargs).case(*args, **kwargs) + + +def flow(*args, **kwargs): + """Decorator ``@flow(...)`` using the default group (see :func:`case`).""" + return _default_group(kwargs).flow(*args, **kwargs) diff --git a/test/kindtest/kindtest/__main__.py b/test/kindtest/kindtest/__main__.py new file mode 100644 index 0000000..0b8253e --- /dev/null +++ b/test/kindtest/kindtest/__main__.py @@ -0,0 +1,10 @@ +"""``python3 -m kindtest`` entry point.""" + +from __future__ import annotations + +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/kindtest/kindtest/cli.py b/test/kindtest/kindtest/cli.py new file mode 100644 index 0000000..510124e --- /dev/null +++ b/test/kindtest/kindtest/cli.py @@ -0,0 +1,165 @@ +"""Command line interface: ``python3 -m kindtest ``. + +Commands: + validate check the config without touching the environment + plan list every case that would run (grouped), no side effects + run execute the selected cases and write the report live + report regenerate the Markdown report from run-manifest.json +""" + +from __future__ import annotations + +import argparse +import json +import sys + +from . import __version__ +from .config import Config, ConfigError +from .model import CATEGORY_ORDER, Case, RunReport, Status +from .registry import RegistryError +from .report import write_report +from .runner import Runner + + +def main(argv: list[str] | None = None) -> int: + common = argparse.ArgumentParser(add_help=False) + common.add_argument("-c", "--config", default=None, help="run config (default: config.yaml)") + + parser = argparse.ArgumentParser(prog="kindtest", description="RLark code-first Kind regression", + parents=[common]) + sub = parser.add_subparsers(dest="command", required=True) + + for name in ("validate", "plan", "run", "report"): + p = sub.add_parser(name, help=f"{name} the run", parents=[common]) + p.add_argument("--only", help="comma-separated ids/groups, e.g. ENV,REG-01; " + "ids match exactly, add a trailing '*' for a prefix (REG-0*)") + p.add_argument("--json", action="store_true", help="machine-readable output") + if name == "run": + p.add_argument("--dry-run", action="store_true", help="print commands without executing") + p.add_argument("--verbose", action="store_true", help="echo every command") + + parser.add_argument("-V", "--version", action="version", version=f"kindtest {__version__}") + args = parser.parse_args(argv) + + try: + config = Config.load(args.config or "config.yaml") + except ConfigError as exc: + print(f"config error: {exc}", file=sys.stderr) + return 2 + + if args.command == "validate": + return _validate(config, args.json) + + only = [t.strip() for t in args.only.split(",")] if getattr(args, "only", None) else None + runner = Runner(config, only=only, dry_run=args.dry_run if args.command == "run" else False, + verbose=args.verbose if args.command == "run" else False) + + if args.command == "plan": + return _plan(config, runner, args.json) + + if args.command == "run": + runner.load_tests() + if args.dry_run: + return _plan(config, runner, args.json) + try: + report = runner.run() + except RegistryError as exc: + print(f"registry error: {exc}", file=sys.stderr) + return 2 + return _summarize(report, args.json) + + if args.command == "report": + return _regenerate(config, args.json) + return 2 + + +def _validate(config: Config, as_json: bool) -> int: + problems = config.validate() + if as_json: + print(json.dumps({"ok": not problems, "problems": problems}, indent=2)) + elif problems: + print("config INVALID:") + for problem in problems: + print(f" - {problem}") + else: + print(f"config OK: theme={config.theme} runId={config.run_id} target={config.target_ref}") + print(f" report dir : {config.report_dir}") + print(f" tests dir : {config.tests_dir}") + return 1 if problems else 0 + + +def _plan(config: Config, runner: Runner, as_json: bool) -> int: + runner.load_tests() + try: + plan = runner.plan() + except RegistryError as exc: + print(f"registry error: {exc}", file=sys.stderr) + return 2 + if as_json: + print(json.dumps([{"group": k, "flows": [ + { + "name": run.name, "kind": run.kind, + "items": [{"id": it.id, "name": it.name, "expect": it.expect} for it in run.items], + } for run in runs + ]} for k, runs in plan], indent=2, ensure_ascii=False)) + return 0 + total = 0 + for key, runs in plan: + items = sum(len(run.items) for run in runs) + print(f"\n== {key} ({len(runs)} 流程 / {items} 观测点) ==") + for run in runs: + if run.kind == "case": + it = run.items[0] + print(f" {it.id:16s} {it.name} [{it.expect}]") + else: + print(f" 流程 {run.name}") + for it in run.items: + print(f" {it.id:16s} {it.name} [{it.expect}]") + total += len(run.items) + print(f"\ntotal: {total} 观测点") + return 0 + + +def _summarize(report: RunReport, as_json: bool) -> int: + counts = report.counts() + if as_json: + print(json.dumps(report.to_dict(), indent=2, ensure_ascii=False)) + else: + print("") + print("=" * 60) + for key in CATEGORY_ORDER: + cc = report.category_counts(key) + if not any(cc.values()): + continue + print(f" {key:<6} 通过 {cc['pass']:>2} 受限 {cc['restricted']:>2} " + f"失败 {cc['fail']:>2} blocked {cc['blocked']:>2}") + print("-" * 60) + print(f" 合计 {len(report.cases)} 项: {counts['pass']} 通过 / {counts['restricted']} 受限 / " + f"{counts['fail']} 分支相关失败 / {counts['blocked']} Blocked / {counts['skipped']} 未执行") + print("=" * 60) + return 1 if counts["fail"] else 0 + + +def _regenerate(config: Config, as_json: bool) -> int: + manifest = config.report_dir / "run-manifest.json" + if not manifest.is_file(): + print(f"no run manifest at {manifest}", file=sys.stderr) + return 1 + data = json.loads(manifest.read_text()) + report = RunReport( + theme=data["theme"], run_id=data["run_id"], base_ref=data["base_ref"], target_ref=data["target_ref"], + started=data["started"], finished=data.get("finished", ""), config=data.get("config", {}), + evidence_dir=data.get("evidence_dir", ""), environment=data.get("environment", {}), + blocked_reason=data.get("blocked_reason", ""), + notes=data.get("notes", {}), + ) + report.cases = [ + Case(id=c["id"], name=c["name"], expected=c.get("expected", ""), actual=c.get("actual", ""), + status=Status(c.get("status", Status.SKIPPED.value)), source=c.get("source", ""), + evidence=list(c.get("evidence", [])), duration_s=c.get("duration_s", 0.0)) + for c in data.get("cases", []) + ] + path = write_report(report, config) + print(json.dumps({"report": str(path), "counts": report.counts()}, indent=2) if as_json + else f"report regenerated: {path}") + return 0 diff --git a/test/kindtest/kindtest/config.py b/test/kindtest/kindtest/config.py new file mode 100644 index 0000000..6b1c75d --- /dev/null +++ b/test/kindtest/kindtest/config.py @@ -0,0 +1,262 @@ +"""Minimal run configuration. + +Only *environment* facts live here - where the clusters are, which images and +named versions identify the stack, which tenant object-name prefix to use and +where to write the report. Anything that describes *what* to verify is written +as a Python test (see ``tests/``). + +Relative paths are resolved against the config file's directory. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + +from .versions import display_image, resolve_image + + +class ConfigError(RuntimeError): + pass + + +_REQUIRED = ("run.id", "clusters.control.kubeconfig", "clusters.data") + + +class Config: + def __init__(self, raw: dict[str, Any], path: Path): + self.raw = raw + self.path = path + self.base = path.resolve().parent + + # ------------------------------------------------------------------ load + @classmethod + def load(cls, path: str | Path) -> "Config": + p = Path(path) + if not p.is_file(): + p = Path.cwd() / path + if not p.is_file(): + raise ConfigError(f"config not found: {path}") + raw = yaml.safe_load(p.read_text()) or {} + if not isinstance(raw, dict): + raise ConfigError(f"config must be a YAML mapping: {p}") + return cls(raw, p) + + # --------------------------------------------------------------- access + def get(self, dotted: str, default: Any = None) -> Any: + node: Any = self.raw + for part in dotted.split("."): + if isinstance(node, dict) and part in node: + node = node[part] + else: + return default + return node + + def require(self, dotted: str) -> Any: + value = self.get(dotted) + if value in (None, ""): + raise ConfigError(f"missing required config key: {dotted}") + return value + + def resolve(self, value: Any) -> Path: + p = Path(str(value)) + if p.is_absolute(): + return p + return (self.base / p).resolve() + + # ------------------------------------------------------------- validate + def validate(self) -> list[str]: + problems: list[str] = [] + for key in _REQUIRED: + if self.get(key) in (None, ""): + problems.append(f"missing required config key: {key}") + data = self.get("clusters.data") or {} + if not isinstance(data, dict) or not data: + problems.append("clusters.data must be a non-empty mapping") + else: + for key, entry in data.items(): + for field in ("name", "clusterId", "kubeconfig"): + if not (entry or {}).get(field): + problems.append(f"clusters.data.{key}.{field} is required") + problems += self._validate_versions() + return problems + + def _validate_versions(self) -> list[str]: + problems: list[str] = [] + defs = self.get("versions.definitions") + if defs is None: + return problems + if not isinstance(defs, dict) or not defs: + problems.append("versions.definitions must be a non-empty mapping") + return problems + for name, entry in defs.items(): + entry = entry or {} + has_images = isinstance(entry.get("images"), dict) and entry["images"] + if not (entry.get("image") or has_images): + problems.append(f"versions.definitions.{name} needs 'images' or 'image'") + initial = self.get("versions.initial") + if initial and initial not in defs: + problems.append(f"versions.initial={initial!r} is not defined in versions.definitions") + for side in ("control", "data"): + for i, target in enumerate(self.get(f"versions.rollout.{side}") or []): + if not (target or {}).get("name"): + problems.append(f"versions.rollout.{side}[{i}].name is required") + return problems + + # ------------------------------------------------------------- identity + @property + def theme(self) -> str: + return str(self.get("run.theme", "kindtest")) + + @property + def run_id(self) -> str: + return str(self.require("run.id")) + + @property + def label_key(self) -> str: + return str(self.get("run.label", "rlark.io/test-run")) + + @property + def domain(self) -> str: + return str(self.get("run.domain", "default")) + + # --------------------------------------------------------------- paths + @property + def report_dir(self) -> Path: + return self.resolve(self.get("report.dir", f"../test-reports/{self.theme}")) + + @property + def report_file(self) -> str: + return str(self.get("report.file", "full-test-result.md")) + + @property + def evidence_dir(self) -> Path: + return self.report_dir / "evidence" + + @property + def manifests_dir(self) -> Path: + return self.resolve(self.get("manifests.dir", self.get("report.dir", f"../test-reports/{self.theme}"))) + + # ------------------------------------------------------------- clusters + @property + def control_kubeconfig(self) -> Path: + return self.resolve(self.require("clusters.control.kubeconfig")) + + @property + def kcp_kubeconfig(self) -> Path: + return self.resolve(self.get("clusters.control.kcp", self.get("clusters.control.kcpKubeconfig", ""))) + + @property + def control_namespace(self) -> str: + return str(self.get("clusters.control.namespace", "rlark-system")) + + def data_clusters(self) -> dict[str, dict[str, Any]]: + return dict(self.get("clusters.data") or {}) + + # --------------------------------------------------------------- images + @property + def task_image(self) -> str: + return str(self.get("images.task", "task:test")) + + @property + def agent_image(self) -> str: + return str(self.get("images.agent", "")) + + @property + def tensorboard_image(self) -> str: + return str(self.get("images.tensorboardStub", "")) + + # ------------------------------------------------------------- versions + @property + def versions_definitions(self) -> dict[str, Any]: + out = self.get("versions.definitions") or {} + return dict(out) if isinstance(out, dict) else {} + + @property + def versions_initial(self) -> str: + initial = str(self.get("versions.initial", "")) + if initial: + return initial + return next(iter(self.versions_definitions), "") + + def version_entry(self, name: str) -> dict[str, Any]: + entry = self.versions_definitions.get(name) or {} + return dict(entry) if isinstance(entry, dict) else {} + + def _version_ref(self, name: str) -> str: + return str(self.version_entry(name).get("ref", "")) + + def version_images(self, name: str) -> dict[str, str]: + """The image slots declared for a version (``image:`` -> ``default``).""" + entry = self.version_entry(name) + if entry.get("image"): + return {"default": str(entry["image"])} + images = entry.get("images") + if isinstance(images, dict) and images: + return {str(k): str(v) for k, v in images.items()} + return {} + + def _version_image(self, name: str) -> str: + """The single concrete image of a version; raises when ambiguous.""" + images = self.version_images(name) + if not images: + return "" + return resolve_image(images, "", what=f"版本 {name!r}") + + # ------------------------------------------------------ baseline / target + # Report-relevant baseline/target come from the named ``versions`` block: + # ``base``/``baseline`` (if present) is the 基线, the version named + # ``target`` or else ``initial`` is the 目标. + def _first_version(self, names: tuple[str, ...]) -> str: + for name in names: + if name in self.versions_definitions: + return name + return "" + + @property + def baseline_ref(self) -> str: + name = self._first_version(("base", "baseline")) + return self._version_ref(name) if name else "" + + @property + def target_ref(self) -> str: + name = self._first_version(("target",)) or self.versions_initial + return self._version_ref(name) if name else "" + + @property + def baseline_image(self) -> str: + name = self._first_version(("base", "baseline")) + return self._version_image(name) if name else "" + + @property + def target_image(self) -> str: + name = self._first_version(("target",)) or self.versions_initial + return self._version_image(name) if name else "" + + @property + def baseline_image_display(self) -> str: + """All image slots of the baseline version, for reports/logs (never raises).""" + return display_image(self.version_images(self._first_version(("base", "baseline")))) + + @property + def target_image_display(self) -> str: + """All image slots of the target version, for reports/logs (never raises).""" + name = self._first_version(("target",)) or self.versions_initial + return display_image(self.version_images(name)) + + # ---------------------------------------------------------------- tests + @property + def tests_dir(self) -> Path: + return self.resolve(self.get("tests.dir", "tests")) + + def test_globs(self) -> list[str]: + include = self.get("tests.include") + if isinstance(include, list) and include: + return [str(x) for x in include] + return ["*.py"] + + @property + def title(self) -> str: + return str(self.get("run.title", f"{self.theme} 功能回归结果")) diff --git a/test/kindtest/kindtest/context.py b/test/kindtest/kindtest/context.py new file mode 100644 index 0000000..355c1e2 --- /dev/null +++ b/test/kindtest/kindtest/context.py @@ -0,0 +1,562 @@ +"""Run context (``t``) - the ergonomic surface tests are written against. + +Two jobs: + +1. **Observation machinery** - the report model is + 测试流程 (flow) → 测试观测点 (observation point) → 测试项 (item). A flow body + wraps each block of checks in ``with t.observe("REG-01"):`` and the assertion + helpers accumulate auditable facts into the active observation. +2. **Basic operation helpers** - small, reusable functions so a test can be + written with almost no boilerplate: ``t.kget``/``t.kapply``/``t.kdelete``/ + ``t.kpatch``/``t.kexec``/``t.klogs``/``t.kwait_pod`` for kubectl, plus + environment shortcuts such as ``t.pod_ready``, ``t.task_uid``, + ``t.job_stopped``, ``t.agent_image``, ``t.ray_active``, ``t.dns_resolve`` + and ``t.tcp_connect``. +""" + +from __future__ import annotations + +import json +import re +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Iterator + +from .kube import Cluster, Kind, Kube +from .model import Status +from .registry import Blocked, Restricted +from .shell import Ran, Shell +from .versions import VersionDeployer, VersionError + + +class EvidenceBook: + """Append-only evidence files under ``/evidence/``. + + The directory is created on demand rather than in ``__init__`` so that a + run can move a previous ``evidence/`` aside (when the target ref changed) + before this run writes its own files. + """ + + def __init__(self, root: Path): + self.root = root + self.files: list[str] = [] + + def ensure(self) -> None: + """(Re)create the evidence directory if it is missing.""" + self.root.mkdir(parents=True, exist_ok=True) + + def add(self, name: str, text: str) -> str: + if name not in self.files: + self.files.append(name) + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as handle: + handle.write(text if text.endswith("\n") else text + "\n") + return name + + +@dataclass +class Observation: + """The recorded result of one 测试观测点 (one report row).""" + + item_id: str + status: Status = Status.PASS + actual: str = "" + facts: list[str] = field(default_factory=list) + evidence: list[str] = field(default_factory=list) + started: float = 0.0 + duration_s: float = 0.0 + + +class Context: + def __init__( + self, + config, + *, + mgmt: Kube, + control: Kube, + clusters: dict[str, Cluster], + shell: Shell, + kind: Kind, + evidence: EvidenceBook, + log: Callable[[str], None] | None = None, + ) -> None: + self.config = config + self.mgmt = mgmt + self.control = control + self.clusters = clusters + self.shell = shell + self.kind = kind + self.evidence_book = evidence + self._log = log + # named versions + rollout (see versions.py); empty when unconfigured + self.versions = VersionDeployer(config, control=control, clusters=clusters, log=self.log) + self.run_id = config.run_id + self.label_key = config.label_key + self.domain = config.domain + self.theme = config.theme + # per-run scratch space shared by tests (e.g. an object name handed on) + self.vars: dict[str, Any] = {} + # runner bookkeeping + self.current_case = "" + self._case_evidence: set[str] = set() + # observation state (see begin_flow / observe) + self.results: dict[str, Observation] = {} + self._declared: list[str] = [] + self._active: Observation | None = None + self.on_item: Callable[[str], None] | None = None + + # --------------------------------------------------------------- logging + def log(self, message: str) -> None: + if self._log is not None: + self._log(message) + + def begin_case(self, case_id: str) -> None: + self.current_case = case_id + self._case_evidence = set() + + # ======================================================== observations + def begin_flow(self, items: list[Any]) -> None: + """Reset observation state for one flow run.""" + self.results = {} + self._declared = [it.id for it in items] + self._active = None + + @contextmanager + def observe(self, item_id: str) -> Iterator[Observation]: + """Open a 测试观测点: checks inside become this item's auditable result. + + Exceptions raised inside the block are captured on the item and do not + abort the rest of the flow; a non-pass verdict was already recorded. + """ + obs = Observation(item_id=item_id, started=time.time()) + previous, previous_case = self._active, self.current_case + self._active = obs + self.current_case = item_id + try: + yield obs + except Blocked as exc: + obs.status, obs.actual = Status.BLOCKED, str(exc) + except Restricted as exc: + obs.status, obs.actual = Status.RESTRICTED, str(exc) + except AssertionError as exc: + obs.status, obs.actual = Status.FAIL, str(exc) or "断言失败" + except Exception as exc: # noqa: BLE001 - any error is a failed observation + obs.status, obs.actual = Status.FAIL, f"{type(exc).__name__}: {exc}" + finally: + obs.duration_s = time.time() - obs.started + if not obs.actual: + obs.actual = "；".join(obs.facts) if obs.facts else "ok" + self.results[item_id] = obs + self._active, self.current_case = previous, previous_case + if self.on_item is not None: + self.on_item(item_id) + + def finish_flow(self, reason: str = "流程未到达该观测点") -> None: + """Mark every declared-but-unobserved item Blocked.""" + for item_id in self._declared: + if item_id not in self.results: + self._record(item_id, Status.BLOCKED, reason) + + def fail_remaining(self, reason: str, status: Status = Status.BLOCKED) -> None: + """Mark every not-yet-observed declared item with ``status``.""" + for item_id in self._declared: + if item_id not in self.results: + self._record(item_id, status, reason) + + def restrict(self, reason: str, item_id: str | None = None) -> None: + """Record an item as 受限 (observed but limited by an existing constraint).""" + self._record(item_id or self.current_case, Status.RESTRICTED, reason) + + def block(self, reason: str, item_id: str | None = None) -> None: + """Record an item as Blocked without raising (flow continues).""" + self._record(item_id or self.current_case, Status.BLOCKED, reason) + + def skip(self, reason: str, item_id: str | None = None) -> None: + self._record(item_id or self.current_case, Status.SKIPPED, reason) + + def _record(self, item_id: str, status: Status, actual: str) -> None: + obs = self.results.get(item_id) or Observation(item_id=item_id) + obs.status, obs.actual = status, actual + self.results[item_id] = obs + if self.on_item is not None: + self.on_item(item_id) + + # --------------------------------------------------------------- handles + def cl(self, key: str) -> Cluster: + return self.clusters[key] + + def data(self, key: str) -> Kube: + return self.clusters[key].data + + def data_keys(self) -> list[str]: + return list(self.clusters) + + def data_ns(self, key: str) -> str: + return self.clusters[key].data_ns + + def mgmt_ns(self, key: str) -> str: + return self.clusters[key].mgmt_ns + + # ------------------------------------------------------------- templating + def render(self, text: str) -> str: + subs = { + "{{runId}}": self.run_id, + "{{labelKey}}": self.label_key, + "{{domain}}": self.domain, + "{{taskImage}}": self.config.task_image, + "{{appImage}}": self.config.agent_image, + "{{agentImage}}": self.config.agent_image, + "{{tensorboardImage}}": self.config.tensorboard_image or "rlinf/tensorboard:2.15.1-rc1", + "{{nodeA}}": str(self.config.get("placement.a", "")), + "{{nodeB}}": str(self.config.get("placement.b", "")), + "{{nodeB2}}": str(self.config.get("placement.b2", "")), + } + # These resolve strictly (a multi-slot version without a default is an + # error), so only evaluate them when the template actually uses them - + # otherwise an unrelated manifest on such a config would fail to render. + lazy = { + "{{baselineImage}}": lambda: self.config.baseline_image, + "{{targetImage}}": lambda: self.config.target_image, + } + for placeholder, resolve in lazy.items(): + if placeholder in text: + text = text.replace(placeholder, resolve()) + for name, version in self.versions.definitions.items(): + subs[f"{{{{image.{name}}}}}"] = version.image + for slot, image in version.images.items(): + subs[f"{{{{image.{name}.{slot}}}}}"] = image + for name, value in self.vars.items(): + subs[f"{{{{{name}}}}}"] = str(value) + for key, value in subs.items(): + text = text.replace(key, value) + return text + + def manifest_text(self, name: str) -> str: + path = self.config.manifests_dir / name + if not path.is_file(): + raise FileNotFoundError(f"manifest not found: {path}") + return self.render(path.read_text()) + + # ================================================== basic ops: kubectl + def _kube(self, on: str, cluster: str | None) -> Kube: + if on == "mgmt": + return self.mgmt + if on == "control": + return self.control + if cluster is None: + raise ValueError("on='data' requires cluster=") + return self.data(cluster) + + def kget(self, kind: str, name: str | None = None, *, on: str = "mgmt", + cluster: str | None = None, ns: str | None = None, + selector: str | None = None, jsonpath: str | None = None, + output: str | None = None, all_namespaces: bool = False) -> Ran: + return self._kube(on, cluster).get( + kind, name, ns=ns, selector=selector, jsonpath=jsonpath, + output=output, all_namespaces=all_namespaces, + ) + + def kjp(self, kind: str, name: str, expr: str, *, on: str = "mgmt", + cluster: str | None = None, ns: str | None = None, default: str = "") -> str: + return self._kube(on, cluster).jp(kind, name, expr, ns=ns, default=default) + + def kjson(self, kind: str, name: str | None = None, *, on: str = "mgmt", + cluster: str | None = None, ns: str | None = None, + selector: str | None = None) -> Any: + return self._kube(on, cluster).json(kind, name, ns=ns, selector=selector) + + def knames(self, kind: str, *, on: str = "mgmt", cluster: str | None = None, + ns: str | None = None, selector: str | None = None, + all_namespaces: bool = False) -> list[str]: + return self._kube(on, cluster).names( + kind, ns=ns, selector=selector, all_namespaces=all_namespaces + ) + + def kexists(self, kind: str, name: str, *, on: str = "mgmt", + cluster: str | None = None, ns: str | None = None) -> bool: + return self._kube(on, cluster).exists(kind, name, ns=ns) + + def kexec(self, cluster: str, pod: str, *argv: str, container: str | None = None, + ns: str | None = None, timeout: int | None = None) -> Ran: + return self.data(cluster).exec(pod, *argv, container=container, + ns=ns or self.data_ns(cluster), timeout=timeout) + + def klogs(self, *, on: str = "mgmt", cluster: str | None = None, ns: str | None = None, + selector: str | None = None, since: str | None = None, + tail: int | None = None) -> Ran: + return self._kube(on, cluster).logs(ns=ns, selector=selector, since=since, tail=tail) + + def kwait_pod(self, cluster: str, pod: str, timeout: int = 300) -> bool: + return self.data(cluster).wait_pod(pod, ns=self.data_ns(cluster), timeout=timeout).ok + + # ---------------------------------------------------- basic ops: effects + def apply(self, manifest: str, *, on: str = "mgmt", cluster: str | None = None, + ns: str | None = None, server_side: bool = False) -> Ran: + result = self._kube(on, cluster).apply(self.render(manifest), ns=ns, server_side=server_side) + if not result.ok: + raise Blocked(f"apply failed rc={result.rc}: {_clip(result.combined())}") + return result + + def apply_manifest(self, name: str, **kw) -> Ran: + return self.apply(self.manifest_text(name), **kw) + + def kdelete(self, kind: str, name: str | None = None, *, on: str = "mgmt", + cluster: str | None = None, ns: str | None = None, + selector: str | None = None, wait: bool = True, + ignore_not_found: bool = True, timeout: int | None = None) -> Ran: + return self._kube(on, cluster).delete( + kind, name, ns=ns, selector=selector, wait=wait, + ignore_not_found=ignore_not_found, timeout=timeout, + ) + + def kpatch(self, kind: str, name: str, patch: Any, *, on: str = "mgmt", + cluster: str | None = None, ns: str | None = None, type: str = "merge") -> Ran: + return self._kube(on, cluster).patch(kind, name, patch, ns=ns, type=type) + + def delete_job(self, name: str, *, wait: bool = True, timeout: int = 180) -> None: + self.mgmt.delete("job", name, ignore_not_found=True, wait=wait, timeout=timeout) + if wait: + self.wait(lambda: not self.mgmt.exists("job", name), timeout=timeout, desc=f"job {name} 消失") + + def wait_job_gone(self, name: str, timeout: int = 180) -> bool: + return self.wait(lambda: not self.mgmt.exists("job", name), timeout=timeout) + + def job_stopped(self, name: str, stopped: bool) -> Ran: + # Job 的停止开关是 spec.suspended；spec.stopped 不存在，API server 会静默剪掉。 + return self.succeeded(self.mgmt.patch("job", name, {"spec": {"suspended": stopped}}), + f"{'stop' if stopped else 'start'} Job {name}") + + def restart_job(self, name: str) -> Ran: + stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + return self.succeeded(self.mgmt.patch("job", name, {"metadata": {"annotations": { + "rlark.io/restarted-at": stamp}}}), f"restart Job {name}") + + # ------------------------------------------------- basic ops: shortcuts + def head_pod(self, job: str, task: str = "head") -> str: + return f"{job}-{task}-0" + + def task_name(self, job: str, task: str = "main") -> str: + return f"{job}-{task}" + + def task_uid(self, cluster: str, task: str) -> str: + return self.mgmt.jp("task", task, "{.metadata.uid}", ns=self.mgmt_ns(cluster)) + + def task_phase(self, cluster: str, task: str) -> str: + return self.mgmt.jp("task", task, "{.status.phase}", ns=self.mgmt_ns(cluster)) + + def pod_ready(self, cluster: str, pod: str) -> bool: + return self.data(cluster).jp( + "pod", pod, "{.status.conditions[?(@.type=='Ready')].status}" + ) == "True" + + def pod_node(self, cluster: str, pod: str) -> str: + return self.data(cluster).jp("pod", pod, "{.spec.nodeName}") + + def exec_ok(self, cluster: str, pod: str, *argv: str) -> bool: + return self.kexec(cluster, pod, *argv).ok + + def dns_resolve(self, cluster: str, pod: str, name: str) -> str: + out = self.kexec( + cluster, pod, "bash", "-c", + f"getent hosts {name} || python3 -c \"import socket;print(socket.gethostbyname('{name}'))\"", + ) + return (out.stdout + out.stderr).strip() + + def tcp_connect(self, cluster: str, pod: str, host: str, port: int) -> bool: + out = self.kexec( + cluster, pod, "bash", "-c", + f"exec 3<>/dev/tcp/{host}/{port} && echo CONNECT_OK", + ) + return "CONNECT_OK" in (out.stdout + out.stderr) + + def ray_active(self, cluster: str, pod: str) -> int: + out = self.kexec(cluster, pod, "ray", "status") + return _active_nodes(out.stdout + out.stderr) + + def agent_image(self, cluster: str) -> str: + return self.cl(cluster).data.jp( + "deploy", "rlark-agent", "{.spec.template.spec.containers[0].image}" + ) + + def agent_ready(self, cluster: str) -> str: + return self.cl(cluster).data.jp( + "deploy", "rlark-agent", "{.status.readyReplicas}/{.spec.replicas}" + ) + + def set_agent_image(self, cluster: str, image: str) -> Ran: + return self.succeeded( + self.cl(cluster).data( + "set", "image", "deploy/rlark-agent", f"rlark-agent={image}", + "-n", self.data_ns(cluster), + ), + f"{cluster} Cluster Agent -> {image}", + ) + + def rollout_agent(self, cluster: str, timeout: int = 240) -> Ran: + return self.succeeded( + self.cl(cluster).data.rollout_status( + "deploy/rlark-agent", ns=self.data_ns(cluster), timeout=timeout + ), + f"{cluster} Cluster Agent rollout", + ) + + # ------------------------------------------------------------- versions + # 配置里定义的命名版本（images/refs）；见 versions.py 与 README“多版本与升级”。 + def version_names(self) -> list[str]: + """Names of every version declared in the config (may be empty).""" + return self.versions.names() + + def version_image(self, name: str, slot: str = "") -> str: + """Container image of a declared version (optionally a named slot).""" + return self.versions.image(name, slot) + + def version_ref(self, name: str) -> str: + """Git ref of a declared version.""" + return self.versions.ref(name) + + def current_version(self) -> str: + """The declared version matching the live target, or the last deployed.""" + return self.versions.detect() or self.versions.current_name + + def version_audit(self, name: str | None = None) -> list[dict]: + """Each rollout target's expected vs live image (default: initial/current).""" + target = name or self.versions.current_name or self.versions.initial + if not target: + raise Restricted("配置未定义 versions，无法核对版本") + return self.versions.audit(target) + + def deploy_version(self, name: str, *, timeout: int = 300, wait: bool = True): + """Deploy a declared version now (idempotent) and record evidence. + + Raises :class:`Restricted` when the config declares no versions, and + :class:`VersionError` for an unknown name or a rollout that never + completes. The framework also deploys ``versions.initial`` once before + the first flow, so a test only needs this to switch mid-run. + """ + if not self.versions.enabled: + raise Restricted("配置未定义 versions，无法部署指定版本") + result = self.versions.deploy(name, timeout=timeout, wait=wait) + self.evidence( + f"version={result.name} images={result.images} changed={result.changed} targets={result.targets}", + f"version-{_safe(result.name)}.txt", + ) + return result + + def upgrade(self, name: str, *, timeout: int = 300, wait: bool = True): + """Move the environment to a declared version (alias of ``deploy_version``).""" + return self.deploy_version(name, timeout=timeout, wait=wait) + + # ----------------------------------------------------------------- waits + def wait(self, predicate: Callable[[], bool], *, timeout: float = 300, interval: float = 5, desc: str = "条件") -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + if predicate(): + return True + except Exception: # noqa: BLE001 - polling must not abort the run + pass + time.sleep(interval) + try: + return bool(predicate()) + except Exception: # noqa: BLE001 + return False + + def require(self, predicate: Callable[[], bool], *, timeout: float = 300, interval: float = 5, desc: str = "条件") -> None: + """Wait for ``predicate`` or raise :class:`Blocked`.""" + if not self.wait(predicate, timeout=timeout, interval=interval): + raise Blocked(f"{desc} 未在 {int(timeout)}s 内满足") + + def wait_pod(self, key: str, pod: str, timeout: int = 300) -> bool: + return self.kwait_pod(key, pod, timeout) + + def sleep(self, seconds: float) -> None: + self.shell.run(["sleep", str(seconds)]) + + # -------------------------------------------------------------- evidence + def evidence(self, text: str, name: str | None = None) -> str: + ev = name or f"case-{_safe(self.current_case)}.txt" + self.evidence_book.add(ev, text) + self._case_evidence.add(ev) + if self._active is not None and ev not in self._active.evidence: + self._active.evidence.append(ev) + return ev + + def case_evidence(self) -> list[str]: + return sorted(self._case_evidence) + + # ----------------------------------------------------------- assertions + # Each helper raises AssertionError on mismatch; on success it records a + # compact fact on the active observation and returns it. + def _fact(self, text: str) -> str: + if self._active is not None: + self._active.facts.append(text) + return text + + def eq(self, got: Any, want: Any, what: str = "值") -> str: + if got != want: + raise AssertionError(f"{what}={got!r} 期望 {want!r}") + return self._fact(f"{what}={got!r}") + + def ne(self, got: Any, want: Any, what: str = "值") -> str: + if got == want: + raise AssertionError(f"{what}={got!r} 不应等于 {want!r}") + return self._fact(f"{what}={got!r}") + + def ge(self, got: Any, want: Any, what: str = "值") -> str: + if not got >= want: + raise AssertionError(f"{what}={got} 期望 ≥{want}") + return self._fact(f"{what}={got}") + + def le(self, got: Any, want: Any, what: str = "值") -> str: + if not got <= want: + raise AssertionError(f"{what}={got} 期望 ≤{want}") + return self._fact(f"{what}={got}") + + def truthy(self, condition: Any, what: str, detail: str = "") -> str: + if not condition: + raise AssertionError(f"{what} 不成立" + (f"（{detail}）" if detail else "")) + return self._fact(what) + + def contains(self, haystack: str, needle: str, what: str = "输出") -> str: + if needle not in (haystack or ""): + raise AssertionError(f"{what} 不含 {needle!r}: {_clip(haystack)}") + return self._fact(f"{what}含 {needle!r}") + + def matches(self, text: str, pattern: str, what: str = "输出") -> str: + if not re.search(pattern, text or ""): + raise AssertionError(f"{what} 未匹配 /{pattern}/: {_clip(text)}") + return self._fact(f"{what} 匹配 /{pattern}/") + + def succeeded(self, ran: Ran, what: str) -> Ran: + if not ran.ok: + raise AssertionError(f"{what} 失败 rc={ran.rc}: {_clip(ran.combined())}") + return ran + + def failed(self, ran: Ran, what: str) -> Ran: + if ran.ok: + raise AssertionError(f"{what} 本应失败却成功: {_clip(ran.combined())}") + return ran + + def nonempty(self, text: str, what: str = "输出") -> str: + if not (text or "").strip(): + raise AssertionError(f"{what} 为空") + return self._fact(f"{what}非空") + + +def _active_nodes(status: str) -> int: + match = re.search(r"(\d+)\s+active", status) + if match: + return int(match.group(1)) + return len(re.findall(r"\bnode_[0-9a-f]+\b", status)) + + +def _safe(name: str) -> str: + return "".join(c if c.isalnum() or c in "-_." else "_" for c in name) or "case" + + +def _clip(text: str, limit: int = 300) -> str: + text = " ".join((text or "").split()) + return text if len(text) <= limit else text[:limit] + "…" diff --git a/test/kindtest/kindtest/kube.py b/test/kindtest/kindtest/kube.py new file mode 100644 index 0000000..c71d798 --- /dev/null +++ b/test/kindtest/kindtest/kube.py @@ -0,0 +1,193 @@ +"""Thin, explicit kubectl/kind handles. + +No implicit kubeconfig is ever used: the management API, the control cluster and +each data cluster have their own :class:`Kube`. A :class:`Cluster` bundles the +data-plane and management-plane views of one data cluster, including the +namespaces each side uses. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Iterable + +from .shell import Ran, Shell + + +class Kube: + """A kubectl bound to one kubeconfig with an optional default namespace.""" + + def __init__(self, shell: Shell, kubeconfig: str | None, *, namespace: str | None = None, label: str = ""): + self.shell = shell + self.kubeconfig = kubeconfig + self.namespace = namespace + self.label = label or (kubeconfig or "?") + + # ------------------------------------------------------------------ raw + def __call__(self, *args: object, **kw) -> Ran: + argv: list[object] = ["kubectl"] + if self.kubeconfig: + argv += ["--kubeconfig", self.kubeconfig] + argv += list(args) + return self.shell.run(argv, **kw) + + def _ns(self, ns: str | None) -> list[str]: + ns = ns if ns is not None else self.namespace + return ["-n", ns] if ns else [] + + # --------------------------------------------------------------- reads + def get( + self, + kind: str, + name: str | None = None, + *, + ns: str | None = None, + selector: str | None = None, + jsonpath: str | None = None, + output: str | None = None, + all_namespaces: bool = False, + ) -> Ran: + args = [*self._ns(ns), "get", kind] + if name: + args.append(name) + if selector: + args += ["-l", selector] + if all_namespaces: + args.append("-A") + if jsonpath: + args += ["-o", f"jsonpath={jsonpath}"] + elif output: + args += ["-o", output] + return self(*args) + + def jp(self, kind: str, name: str, expr: str, *, ns: str | None = None, default: str = "") -> str: + """One jsonpath value, or ``default`` when the object/expression is missing.""" + ran = self.get(kind, name, ns=ns, jsonpath=expr) + return ran.out if ran.ok and ran.out else default + + def probe(self, timeout: float = 15) -> Ran: + """Cheap reachability check (``GET /readyz``) for the preflight. + + Distinguishes an unreachable cluster (non-zero rc) from an object that + is simply absent, which :meth:`jp` deliberately collapses to ``default``. + """ + return self("get", "--raw=/readyz", timeout=timeout) + + def json(self, kind: str, name: str | None = None, *, ns: str | None = None, selector: str | None = None) -> Any: + ran = self.get(kind, name, ns=ns, selector=selector, output="json") + if not ran.ok: + raise RuntimeError(f"kubectl get {kind} failed: {ran.err[:200]}") + return json.loads(ran.out) if ran.out else None + + def exists(self, kind: str, name: str, *, ns: str | None = None) -> bool: + return self.get(kind, name, ns=ns, output="name").ok + + def names(self, kind: str, *, ns: str | None = None, selector: str | None = None, + all_namespaces: bool = False) -> list[str]: + """Object names (``name`` only, no kind prefix) for a list query.""" + ran = self.get(kind, ns=ns, selector=selector, output="name", all_namespaces=all_namespaces) + return [ln.split("/")[-1] for ln in ran.stdout.splitlines() if ln.strip()] + + # ------------------------------------------------------------ mutations + def apply(self, manifest: str, *, ns: str | None = None, server_side: bool = False) -> Ran: + args = [*self._ns(ns), "apply", "-f", "-"] + if server_side: + args.append("--server-side") + return self(*args, input=manifest) + + def delete( + self, + kind: str, + name: str | None = None, + *, + ns: str | None = None, + selector: str | None = None, + wait: bool = True, + ignore_not_found: bool = True, + timeout: int | None = None, + ) -> Ran: + args = [*self._ns(ns), "delete", kind] + if name: + args.append(name) + if selector: + args += ["-l", selector] + if ignore_not_found: + args.append("--ignore-not-found") + if not wait: + args.append("--wait=false") + if timeout: + args.append(f"--timeout={timeout}s") + return self(*args) + + def patch(self, kind: str, name: str, patch: Any, *, ns: str | None = None, type: str = "merge") -> Ran: + text = patch if isinstance(patch, str) else json.dumps(patch) + return self(*self._ns(ns), "patch", kind, name, f"--type={type}", "-p", text) + + def exec(self, pod: str, *argv: str, container: str | None = None, ns: str | None = None, timeout: int | None = None) -> Ran: + args = [*self._ns(ns), "exec", pod] + if container: + args += ["-c", container] + args += ["--", *argv] + return self(*args, timeout=timeout) + + def logs(self, *, ns: str | None = None, selector: str | None = None, since: str | None = None, tail: int | None = None) -> Ran: + args = [*self._ns(ns), "logs"] + if selector: + args += ["-l", selector] + if since: + args.append(f"--since={since}") + if tail: + args.append(f"--tail={tail}") + return self(*args) + + def rollout_status(self, target: str, *, ns: str | None = None, timeout: int = 240) -> Ran: + return self(*self._ns(ns), "rollout", "status", target, f"--timeout={timeout}s") + + def wait_pod(self, pod: str, *, ns: str | None = None, timeout: int = 300) -> Ran: + return self(*self._ns(ns), "wait", "--for=condition=Ready", f"pod/{pod}", f"--timeout={timeout}s") + + +@dataclass +class Cluster: + key: str + name: str + cluster_id: str + data: Kube + mgmt: Kube + data_ns: str = "rlark-system" + + @property + def mgmt_ns(self) -> str: + return self.cluster_id + + +class Kind: + """Host-level ``kind`` helpers (node containers / image loading).""" + + def __init__(self, shell: Shell): + self.shell = shell + + def clusters(self) -> list[str]: + return [n for n in self.shell.run(["kind", "get", "clusters"]).stdout.split() if n] + + def nodes(self, cluster: str | None = None) -> list[str]: + argv = ["kind", "get", "nodes"] + if cluster: + argv += ["--name", cluster] + return [n for n in self.shell.run(argv).stdout.split() if n] + + def containers_up(self) -> list[str]: + ran = self.shell.run(["docker", "ps", "--format", "{{.Names}}\t{{.Status}}"]) + return [ln.split("\t", 1)[0] for ln in ran.stdout.splitlines() if "\tUp" in ln] + + def image_id(self, image: str) -> str: + ran = self.shell.run(["docker", "image", "inspect", "-f", "{{.Id}}", image], quiet=True) + return ran.out if ran.ok else "" + + def load(self, cluster: str, images: Iterable[str]) -> Ran: + return self.shell.run(["kind", "load", "docker-image", *images, "--name", cluster], timeout=1800) + + def node_has_image(self, node: str, needle: str) -> bool: + ran = self.shell.run(["docker", "exec", node, "crictl", "images", "-q"], quiet=True) + return needle in ran.stdout diff --git a/test/kindtest/kindtest/model.py b/test/kindtest/kindtest/model.py new file mode 100644 index 0000000..1892ad8 --- /dev/null +++ b/test/kindtest/kindtest/model.py @@ -0,0 +1,144 @@ +"""Result data model: one :class:`Case` per reported matrix row. + +The status vocabulary follows the project's reporting conventions so the +generated report never silently turns a restricted/blocked observation into a +pass. +""" + +from __future__ import annotations + +import enum +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone + + +def utcnow() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class Status(str, enum.Enum): + PASS = "通过" + FAIL = "失败" + RESTRICTED = "受限" + BLOCKED = "Blocked" + SKIPPED = "未执行" + + @property + def counted(self) -> bool: + """True when the status represents a recorded observation.""" + return self is not Status.SKIPPED + + +@dataclass +class Case: + id: str + name: str + expected: str = "" + actual: str = "" + status: Status = Status.SKIPPED + source: str = "Kind" + evidence: list[str] = field(default_factory=list) + duration_s: float = 0.0 + + @property + def category(self) -> str: + return self.id.split("-", 1)[0].split("[", 1)[0] + + def to_dict(self) -> dict: + data = asdict(self) + data["status"] = self.status.value + data["category"] = self.category + return data + + +CATEGORY_ORDER: list[str] = [ + "ENV", + "BUILD", + "UNIT", + "BIND", + "MIG", + "CRD", + "REG", + "TASK", + "SSH", + "VALID", + "OBS", + "CLEAN", +] + +CATEGORY_TITLE: dict[str, str] = { + "ENV": "环境和组件健康", + "BUILD": "镜像构建和加载", + "UNIT": "单元测试和静态检查", + "BIND": "新契约定向测试", + "MIG": "在途升级和不变量", + "CRD": "CRD Schema", + "REG": "基本功能回归", + "TASK": "生命周期和错误恢复", + "SSH": "工作负载 SSH 访问", + "VALID": "校验和拒绝", + "OBS": "可观测性和 root metadata", + "CLEAN": "清理和最终状态", +} + + +@dataclass +class RunReport: + theme: str + run_id: str + base_ref: str + target_ref: str + started: str + finished: str = "" + config: dict = field(default_factory=dict) + cases: list[Case] = field(default_factory=list) + evidence_dir: str = "" + environment: dict = field(default_factory=dict) + blocked_reason: str = "" + notes: dict = field(default_factory=dict) + + def counts(self) -> dict[str, int]: + out = {"pass": 0, "fail": 0, "restricted": 0, "blocked": 0, "skipped": 0} + for case in self.cases: + out[_status_key(case.status)] += 1 + return out + + def by_category(self) -> dict[str, list[Case]]: + grouped: dict[str, list[Case]] = {} + for case in self.cases: + grouped.setdefault(case.category, []).append(case) + for cases in grouped.values(): + cases.sort(key=lambda c: _id_sort_key(c.id)) + return grouped + + def category_counts(self, key: str) -> dict[str, int]: + out = {"pass": 0, "fail": 0, "restricted": 0, "blocked": 0, "skipped": 0} + for case in self.cases: + if case.category == key: + out[_status_key(case.status)] += 1 + return out + + def to_dict(self) -> dict: + data = asdict(self) + data["cases"] = [c.to_dict() for c in self.cases] + data["counts"] = self.counts() + return data + + +def _status_key(status: Status) -> str: + return { + Status.PASS: "pass", + Status.FAIL: "fail", + Status.RESTRICTED: "restricted", + Status.BLOCKED: "blocked", + Status.SKIPPED: "skipped", + }[status] + + +def _id_sort_key(case_id: str): + prefix, _, suffix = case_id.partition("-") + numeric = suffix.split("[", 1)[0] + try: + return (prefix, 0, int(numeric), suffix) + except ValueError: + return (prefix, 1, 0, suffix) diff --git a/test/kindtest/kindtest/registry.py b/test/kindtest/kindtest/registry.py new file mode 100644 index 0000000..d5a0d76 --- /dev/null +++ b/test/kindtest/kindtest/registry.py @@ -0,0 +1,280 @@ +"""Test authoring registry - 测试流程 (flows), 观测点 (observations), 测试项 (items). + +The public surface mirrors the vocabulary used in the reports: + +* a **group** (``ENV``/``REG``/...) is a reporting category; +* a **flow** (测试流程) is a decorated procedure that runs *once* and may touch + the environment; it is defined with :meth:`Group.flow` and may declare its + observation points up front via ``items=`` and/or ``@group.item(...)``; +* an **observation point** (测试观测点) is a block inside a flow that evaluates + one or more **items** (测试项) - the rows of the report - and recorded with + ``with t.observe("REG-01"):``; +* a **case** is a one-line sugar for "a flow with a single observation point", + registered with :meth:`Group.case` (optionally parametrized). + +Flow functions are ordinary Python functions. They receive the run context +(``t``) first, then any fixtures/parameters they name in their signature. A +flow/case passes by raising nothing; it fails by raising. ``Blocked`` and +``Restricted`` raise to record those non-pass verdicts - outside an observation +they abort the flow and mark every not-yet-observed declared item. +""" + +from __future__ import annotations + +import inspect +from dataclasses import dataclass, field +from typing import Any, Callable, Iterable, Mapping, Sequence + + +class Blocked(Exception): + """A precondition could not be established; the item is reported Blocked.""" + + +class Restricted(Exception): + """Observed, but limited by an existing/environmental constraint.""" + + +class RegistryError(RuntimeError): + """The registered tests are invalid, e.g. two items share an id.""" + + +@dataclass +class FixtureDef: + name: str + fn: Callable[..., Any] + scope: str # "group" | "case" + + +@dataclass +class Item: + """One test item - a single row in the report matrix (测试项).""" + + id: str + name: str + expect: str = "" + source: str = "Kind" + + +@dataclass +class FlowSpec: + """A registered procedure (测试流程) or single-observation case.""" + + group: "Group" + name: str + items: list[Item] + fn: Callable[..., Any] + kind: str = "flow" # "flow" | "case" + params: list[dict] | None = None + ids: list[str] | None = None + + +@dataclass +class FlowRun: + """A concrete execution of a flow: one params combination, expanded items.""" + + group_key: str + group_title: str + name: str + kind: str + fn: Callable[..., Any] + items: list[Item] = field(default_factory=list) + values: dict = field(default_factory=dict) + + +class Group: + def __init__(self, key: str, title: str, order: int): + self.key = key + self.title = title + self.order = order + self.flows: list[FlowSpec] = [] + + # ------------------------------------------------------------- authoring + def flow( + self, + name: str, + items: Sequence[Any] | None = None, + *, + source: str = "Kind", + params: list[dict] | None = None, + ids: list[str] | None = None, + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Register a procedure (测试流程) run once, with declared observation points. + + ``items`` may mix tuples ``(id, name[, expect])``, mappings with those + keys, or :class:`Item`. Additional observation points may be declared by + stacking ``@group.item(...)`` under the ``@group.flow(...)`` decorator. + Items are ordered by id so the report matrix is deterministic. + """ + + declared = _normalize_items(items or [], source) + + def deco(fn: Callable[..., Any]) -> Callable[..., Any]: + extra = _normalize_items(getattr(fn, "_kindtest_items", []), source) + all_items = sorted(declared + extra, key=lambda it: _item_sort_key(it.id)) + if not all_items: + raise ValueError( + f"流程 {name!r} 未声明任何观测点：使用 items= 或 @group.item(...)" + ) + self.flows.append( + FlowSpec(self, name, all_items, fn, kind="flow", params=params, ids=ids) + ) + return fn + + return deco + + def item( + self, id: str, name: str, *, expect: str = "", source: str = "Kind" + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Declare one observation point of the flow decorated below/above.""" + + def deco(fn: Callable[..., Any]) -> Callable[..., Any]: + existing = list(getattr(fn, "_kindtest_items", [])) + existing.append(Item(id, name, expect, source)) + fn._kindtest_items = existing + return fn + + return deco + + def case( + self, + id: str, + name: str, + *, + expect: str = "", + source: str = "Kind", + params: list[dict] | None = None, + ids: list[str] | None = None, + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Register a single-observation case (sugar for a one-item flow).""" + + def deco(fn: Callable[..., Any]) -> Callable[..., Any]: + item = Item(id, name, expect, source) + self.flows.append( + FlowSpec(self, name, [item], fn, kind="case", params=params, ids=ids) + ) + return fn + + return deco + + # ------------------------------------------------------------------ plan + def collect(self) -> list[FlowRun]: + runs: list[FlowRun] = [] + for spec in self.flows: + runs += _expand(spec) + return runs + + +# -------------------------------------------------------------------------- +# global registries +# -------------------------------------------------------------------------- +GROUPS: dict[str, Group] = {} +_GROUP_ORDER: list[str] = [] +FIXTURES: dict[str, FixtureDef] = {} + + +def group(key: str, title: str | None = None) -> Group: + if key not in GROUPS: + GROUPS[key] = Group(key, title or key, len(_GROUP_ORDER)) + _GROUP_ORDER.append(key) + return GROUPS[key] + + +def fixture(*, scope: str = "group", name: str | None = None) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + if scope not in ("group", "case"): + raise ValueError("fixture scope must be 'group' or 'case'") + + def deco(fn: Callable[..., Any]) -> Callable[..., Any]: + FIXTURES[name or fn.__name__] = FixtureDef(name or fn.__name__, fn, scope) + return fn + + return deco + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- +def _normalize_items(raw: Iterable[Any], default_source: str) -> list[Item]: + out: list[Item] = [] + for entry in raw: + if isinstance(entry, Item): + out.append(entry) + elif isinstance(entry, Mapping): + out.append( + Item( + str(entry["id"]), + str(entry.get("name", entry["id"])), + str(entry.get("expect", "")), + str(entry.get("source", default_source)), + ) + ) + elif isinstance(entry, (tuple, list)): + seq = list(entry) + out.append( + Item( + str(seq[0]), + str(seq[1]) if len(seq) > 1 else str(seq[0]), + str(seq[2]) if len(seq) > 2 else "", + str(seq[3]) if len(seq) > 3 else default_source, + ) + ) + else: + out.append(Item(str(entry), str(entry), "", default_source)) + return out + + +def _item_sort_key(item_id: str): + head, _, tail = item_id.partition("-") + numeric = tail.split("[", 1)[0] + try: + return (head, 0, int(numeric), item_id) + except ValueError: + return (head, 1, 0, item_id) + + +def _expand(spec: FlowSpec) -> list[FlowRun]: + if not spec.params: + return [ + FlowRun( + spec.group.key, spec.group.title, spec.name, spec.kind, spec.fn, + list(spec.items), {}, + ) + ] + runs: list[FlowRun] = [] + suffix_all = len(spec.params) > 1 or bool(spec.ids) + for index, values in enumerate(spec.params): + label = spec.ids[index] if spec.ids and index < len(spec.ids) else str(index + 1) + suffix = f"[{label}]" if suffix_all else "" + items = [Item(it.id + suffix, it.name, it.expect, it.source) for it in spec.items] + runs.append( + FlowRun( + spec.group.key, spec.group.title, spec.name, spec.kind, spec.fn, + items, dict(values), + ) + ) + return runs + + +def collect(selected_groups: list[str] | None = None) -> list[FlowRun]: + """All registered flow runs in canonical category order.""" + runs: list[FlowRun] = [] + for key in ordered_group_keys(selected_groups): + runs += GROUPS[key].collect() + return runs + + +def ordered_group_keys(selected: list[str] | None = None) -> list[str]: + from .model import CATEGORY_ORDER + + def rank(key: str) -> tuple[int, int]: + canonical = CATEGORY_ORDER.index(key) if key in CATEGORY_ORDER else len(CATEGORY_ORDER) + return (canonical, GROUPS[key].order) + + keys = [k for k in _GROUP_ORDER if not selected or k in selected] + return sorted(keys, key=rank) + + +def case_signature(fn: Callable[..., Any]) -> list[str]: + """Parameter names after the leading context parameter.""" + params = [p.name for p in inspect.signature(fn).parameters.values() + if p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY)] + return params[1:] diff --git a/test/kindtest/kindtest/report.py b/test/kindtest/kindtest/report.py new file mode 100644 index 0000000..711245d --- /dev/null +++ b/test/kindtest/kindtest/report.py @@ -0,0 +1,234 @@ +"""Markdown report generation aligned with the project's reporting conventions.""" + +from __future__ import annotations + +from pathlib import Path + +from .config import Config +from .model import CATEGORY_ORDER as DEFAULT_CATEGORY_ORDER +from .model import CATEGORY_TITLE as DEFAULT_CATEGORY_TITLE +from .model import Case, RunReport, Status + + +def write_report(report: RunReport, config: Config) -> Path: + path = config.report_dir / config.report_file + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render(report, config)) + return path + + +def render(report: RunReport, config: Config) -> str: + counts = report.counts() + lines: list[str] = [f"# {config.title}", "", _banner(report, counts), ""] + if report.blocked_reason: + lines += _blocked(report) + lines += _criteria(config) + lines += _baseline(report, config) + lines += _summary(report, config) + lines += _details(report, config) + lines += _findings(config) + lines += _conclusion(report, config, counts) + lines += _environment(report) + lines += _evidence_index(report) + lines.append("") + lines.append(_footer()) + return "\n".join(lines) + "\n" + + +def _banner(report: RunReport, counts: dict[str, int]) -> str: + status = "已完成" if report.finished else "进行中（实时更新）" + stamp = report.finished or report.started + total = len(report.cases) + done = total - counts["skipped"] + where = f"目标 `{report.target_ref}`。" if report.target_ref else "" + if report.base_ref: + where = f"基线 `{report.base_ref}`，目标 `{report.target_ref}`。" + return ( + f"> 状态：**{status}**。{where}" + f"已记录 {done}/{total} 项：{counts['pass']} 通过 / {counts['restricted']} 受限 / " + f"{counts['fail']} 分支相关失败 / {counts['blocked']} Blocked / {counts['skipped']} 未执行。" + f"（更新于 {stamp}）" + ) + + +def _blocked(report: RunReport) -> list[str]: + return [ + "", + f"> ⛔ **运行未开始（环境/setup 未通过）**：{report.blocked_reason}", + ">", + "> 这是前置条件问题，不是测试项的失败；修复环境后重跑，不要按失败项排查。", + ] + + +def _criteria(config: Config) -> list[str]: + out = ["", "## 判据结论", ""] + items = config.get("notes.criteria") or [] + if items: + out += [f"- {x}" for x in items] + else: + out.append("- （在配置 `notes.criteria` 中填写判据结论）") + return out + + +def _baseline(report: RunReport, config: Config) -> list[str]: + out = ["", "## 测试基线", "", "| 项目 | 值 |", "| --- | --- |"] + rows = [] + if report.base_ref: + rows.append(("基线版本", f"`{report.base_ref}`")) + rows.append(("目标版本", f"`{report.target_ref}`")) + if config.baseline_image_display: + rows.append(("基线镜像", f"`{config.baseline_image_display}`")) + rows.append(("目标镜像", f"`{config.target_image_display}`")) + rows.append(("Task 镜像", f"`{config.task_image}`")) + if config.versions_definitions: + rows.append(("初始版本", f"`{config.versions_initial}`")) + rows.append(("定义版本", _versions_cell(config))) + rows += [ + ("控制集群", f"`{config.get('clusters.control.name', '')}`"), + ("数据集群", "、".join(f"`{v.get('name')}`" for v in config.data_clusters().values())), + ("测试标签", f"`{config.label_key}={config.run_id}*`；名称前缀 `{config.run_id}*`"), + ] + excluded = config.get("notes.excluded") + if excluded: + rows.append(("排除集群", str(excluded))) + out += [f"| {k} | {v} |" for k, v in rows] + return out + + +def _versions_cell(config: Config) -> str: + initial = config.versions_initial + parts = [] + for name, entry in config.versions_definitions.items(): + entry = entry or {} + mark = "（初始）" if name == initial else "" + images = entry.get("images") + if isinstance(images, dict) and images: + shown = "、".join(f"{k}=`{v}`" for k, v in images.items()) + else: + shown = f"`{entry.get('image', '')}`" + detail = f"`{name}`{mark}: {shown}" + if entry.get("ref"): + detail += f" @ `{entry['ref']}`" + parts.append(detail) + return "；".join(parts) + + +def _summary(report: RunReport, config: Config) -> list[str]: + out = ["", "## 总体汇总", "", + "| 测试类别 | 通过 | 受限 | 分支相关失败 | Blocked | 未执行 | 结论 |", + "| --- | ---: | ---: | ---: | ---: | ---: | --- |"] + order, titles = _category_plan(report) + total = {"pass": 0, "restricted": 0, "fail": 0, "blocked": 0, "skipped": 0} + for key in order: + cc = report.category_counts(key) + if not any(cc.values()): + continue + for k in total: + total[k] += cc[k] + out.append( + f"| {_title(key, titles)} ({key}) | {cc['pass']} | {cc['restricted']} | {cc['fail']} | " + f"{cc['blocked']} | {cc['skipped']} | {_verdict(cc)} |" + ) + out.append( + f"| **合计** | **{total['pass']}** | **{total['restricted']}** | **{total['fail']}** | " + f"**{total['blocked']}** | **{total['skipped']}** | " + f"**{len(report.cases)} 项：{total['pass']} 通过 / {total['restricted']} 受限 / {total['fail']} 分支相关失败** |" + ) + return out + + +def _details(report: RunReport, config: Config) -> list[str]: + out: list[str] = [] + grouped = report.by_category() + order, titles = _category_plan(report) + for key in order: + cases = grouped.get(key) + if not cases: + continue + out += ["", f"## {_title(key, titles)}", "", + "| ID | 测试项 | 预期 | 实际结果 | 来源 | 状态 |", + "| --- | --- | --- | --- | --- | --- |"] + for case in cases: + actual = case.actual or "" + if case.evidence: + actual = f"{actual}（`{', '.join(case.evidence)}`）" + out.append( + f"| {case.id} | {case.name} | {case.expected} | {actual} | {case.source} | {case.status.value} |" + ) + return out + + +def _findings(config: Config) -> list[str]: + items = config.get("notes.findings") or [] + if not items: + return [] + out = ["", "## 发现的问题", ""] + out += [f"{i}. {x}" for i, x in enumerate(items, start=1)] + return out + + +def _conclusion(report: RunReport, config: Config, counts: dict[str, int]) -> list[str]: + out = ["", "## 最终结论", ""] + if report.blocked_reason: + out += [f"**运行未开始（环境/setup 未通过）**：{report.blocked_reason}", ""] + for para in config.get("notes.conclusion") or []: + out += [para, ""] + out.append( + f"**汇总：矩阵 {len(report.cases)} 项 = {counts['pass']} 通过 / {counts['restricted']} 受限 / " + f"{counts['fail']} 分支相关失败 / {counts['blocked']} Blocked / {counts['skipped']} 未执行。**" + ) + return out + + +def _environment(report: RunReport) -> list[str]: + env = report.environment or {} + if not env: + return [] + out = ["", "## 最终环境状态", "", "| 项目 | 最终值 |", "| --- | --- |"] + out += [f"| {k} | {v} |" for k, v in env.items()] + return out + + +def _evidence_index(report: RunReport) -> list[str]: + usage: dict[str, list[str]] = {} + for case in report.cases: + for ev in case.evidence: + usage.setdefault(ev, []).append(case.id) + if not usage: + return [] + out = ["", "## 附：证据索引", "", "| 证据文件 | 覆盖项 |", "| --- | --- |"] + for name in sorted(usage): + out.append(f"| `{name}` | {', '.join(sorted(set(usage[name])))} |") + return out + + +def _footer() -> str: + return ( + "> 证据来源标注：Kind=真实数据面端到端；单元测试=确定性 go test；源码=静态检查；" + "构建=镜像构建与 digest；Git=版本核对；环境=宿主资源。" + ) + + +def _category_plan(report: RunReport) -> tuple[list[str], dict[str, str]]: + present = {case.category for case in report.cases} + order = list(DEFAULT_CATEGORY_ORDER) + for key in sorted(present): + if key not in order: + order.append(key) + return order, dict(DEFAULT_CATEGORY_TITLE) + + +def _title(key: str, titles: dict[str, str]) -> str: + return titles.get(key) or key + + +def _verdict(cc: dict[str, int]) -> str: + if cc["fail"]: + return "存在分支相关失败" + if cc["blocked"]: + return "存在 Blocked" + if cc["restricted"]: + return "受限" + if cc["skipped"]: + return "部分未执行" + return "全部通过" diff --git a/test/kindtest/kindtest/runner.py b/test/kindtest/kindtest/runner.py new file mode 100644 index 0000000..91f4944 --- /dev/null +++ b/test/kindtest/kindtest/runner.py @@ -0,0 +1,447 @@ +"""Execution engine: full-matrix-first, flow execution, live report. + +The report matrix is built up front from the declared items (every 测试项 starts +as 未执行) and refreshed after each 测试观测点. Group-scoped fixtures are created +once per group and torn down after it, case-scoped fixtures per flow run. +""" + +from __future__ import annotations + +import importlib.util +import inspect +import json +import shutil +import sys +import time +import traceback +from pathlib import Path +from typing import Any, Callable, Iterable + +from . import registry +from .config import Config +from .context import Context, EvidenceBook +from .kube import Cluster, Kind, Kube +from .model import Case, RunReport, Status, utcnow +from .registry import Blocked, FIXTURES, FlowRun, RegistryError, Restricted +from .report import write_report +from .shell import Shell + + +class _Scope: + """Cache + teardown stack for one fixture scope (a group or a flow run).""" + + def __init__(self) -> None: + self.cache: dict[str, Any] = {} + self.errors: dict[str, BaseException] = {} + self.teardowns: list[Any] = [] + + def close(self, log: Callable[[str], None]) -> None: + for gen in reversed(self.teardowns): + try: + next(gen) + except StopIteration: + pass + except Exception as exc: # noqa: BLE001 - teardown must not abort a run + log(f"fixture teardown error: {type(exc).__name__}: {exc}") + self.teardowns.clear() + self.cache.clear() + + +class Runner: + def __init__(self, config: Config, *, only: Iterable[str] | None = None, dry_run: bool = False, + verbose: bool = False) -> None: + self.config = config + self.only = [t for t in (only or []) if t] + self.dry_run = dry_run + self.verbose = verbose + self.report_dir = config.report_dir + self.report_dir.mkdir(parents=True, exist_ok=True) + self.log_path = self.report_dir / f"run-{config.run_id}.log" + self._log_handle = self.log_path.open("a") + self.shell = Shell(dry_run=dry_run, verbose=verbose, log=self._log) + self.evidence = EvidenceBook(config.evidence_dir) + self.kind = Kind(self.shell) + self.ctx = self._build_context() + self.ctx.on_item = self._refresh_item + self.live: dict[str, Case] = {} + self.started = "" + self.blocked_reason = "" + + # ------------------------------------------------------------- wiring + def _build_context(self) -> Context: + cfg = self.config + clusters: dict[str, Cluster] = {} + for key, entry in cfg.data_clusters().items(): + kubeconfig = str(cfg.resolve(entry["kubeconfig"])) + data_ns = str(entry.get("namespace", "rlark-system")) + clusters[key] = Cluster( + key=key, + name=str(entry["name"]), + cluster_id=str(entry["clusterId"]), + data=Kube(self.shell, kubeconfig, namespace=data_ns, label=str(entry["name"])), + mgmt=Kube(self.shell, str(cfg.kcp_kubeconfig), namespace=str(entry["clusterId"])), + data_ns=data_ns, + ) + mgmt = Kube(self.shell, str(cfg.kcp_kubeconfig), namespace=None, label="kcp") + control = Kube(self.shell, str(cfg.control_kubeconfig), namespace=cfg.control_namespace, label="control") + return Context(cfg, mgmt=mgmt, control=control, clusters=clusters, shell=self.shell, + kind=self.kind, evidence=self.evidence, log=self._log) + + # ------------------------------------------------------------- loading + def load_tests(self) -> list[str]: + modules: list[str] = [] + for pattern in self.config.test_globs(): + for path in sorted(self.config.tests_dir.glob(pattern)): + if path.name.startswith("_"): + continue + name = f"kindtest_tests_{path.stem}" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + continue + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + modules.append(path.name) + return modules + + # --------------------------------------------------------------- plan + def runs(self) -> list[FlowRun]: + registered = registry.collect() + self._assert_unique_item_ids(registered) + out: list[FlowRun] = [] + for run in registered: + if not self.only: + out.append(run) + continue + if run.group_key in self.only or run.name in self.only: + out.append(run) + continue + items = [it for it in run.items if self._item_selected(it.id)] + if items: + out.append( + FlowRun(run.group_key, run.group_title, run.name, run.kind, run.fn, + items, dict(run.values)) + ) + return out + + @staticmethod + def _assert_unique_item_ids(runs: list[FlowRun]) -> None: + """Reject duplicate item ids before they collapse into one report row. + + The live matrix and the final report are keyed by item id, so a second + item with the same id would silently overwrite the first. Surface the + mistake (and its declaring flows) instead. + """ + owner: dict[str, str] = {} + dupes: dict[str, list[str]] = {} + for run in runs: + for it in run.items: + if it.id in owner: + dupes.setdefault(it.id, [owner[it.id]]).append(run.name) + else: + owner[it.id] = run.name + if dupes: + detail = "；".join(f"{i}（{'、'.join(names)}）" for i, names in dupes.items()) + raise RegistryError(f"测试项 id 重复，报告矩阵会互相覆盖：{detail}") + + def _item_selected(self, item_id: str) -> bool: + """``--only`` matches ids exactly; a trailing ``*`` selects a prefix. + + So ``REG-01`` selects one item while ``REG-0*`` selects ``REG-01``… + ``REG-09``. A parametrized id also matches when its base (the part + before ``[``) is given. + """ + base = item_id.split("[", 1)[0] + for token in self.only: + if token.endswith("*"): + prefix = token[:-1] + if item_id.startswith(prefix) or base.startswith(prefix): + return True + elif token in (item_id, base): + return True + return False + + def plan(self) -> list[tuple[str, list[FlowRun]]]: + out: list[tuple[str, list[FlowRun]]] = [] + by_key: dict[str, list[FlowRun]] = {} + for run in self.runs(): + by_key.setdefault(run.group_key, []).append(run) + for key in registry.ordered_group_keys(): + if key in by_key: + out.append((key, by_key[key])) + return out + + # ---------------------------------------------------------------- run + def run(self) -> RunReport: + self._archive_previous() + self.evidence.ensure() + self.started = utcnow() + runs = self.runs() + # full matrix first: every declared item starts as 未执行 + self.live = { + it.id: Case(id=it.id, name=it.name, expected=it.expect, actual="", + status=Status.SKIPPED, source=it.source) + for run in runs for it in run.items + } + self._write_live(finished="") + try: + self._preflight() + self._deploy_initial_version() + except Blocked as exc: + return self._block_run(str(exc)) + + group_key: str | None = None + group_scope: _Scope | None = None + for run in runs: + if run.group_key != group_key: + if group_scope is not None: + group_scope.close(self._log) + group_key = run.group_key + group_scope = _Scope() + self._log(f"===== GROUP {group_key} =====") + self._run_flow(run, group_scope) + if group_scope is not None: + group_scope.close(self._log) + + report = self._write_live(finished=utcnow(), environment=self._environment()) + self._log_handle.close() + return report + + def _deploy_initial_version(self) -> None: + """Deploy ``versions.initial`` once, before the first flow touches the env.""" + versions = self.ctx.versions + if not versions.enabled: + return + initial = versions.initial + if self.dry_run: + self._log( + f"[dry-run] initial version {initial} " + f"(images={versions.version(initial).images})" + ) + return + if not versions.rollout_configured: + self._log( + f"versions defined ({'、'.join(versions.names())}) but versions.rollout " + f"未配置；跳过初始版本 {initial} 的部署" + ) + return + self._log( + f"===== INITIAL VERSION {initial} " + f"(images={versions.version(initial).images}) =====" + ) + try: + self.ctx.deploy_version(initial) + except Exception as exc: # noqa: BLE001 - a failed setup blocks the run + raise Blocked( + f"初始版本 {initial} 部署失败：{type(exc).__name__}: {exc}" + ) from exc + + def _probe_targets(self) -> list[tuple[str, Kube]]: + targets: list[tuple[str, Kube]] = [ + ("KCP", self.ctx.mgmt), + ("控制集群", self.ctx.control), + ] + for key in self.ctx.data_keys(): + targets.append((f"数据集群 {self.ctx.cl(key).name}", self.ctx.cl(key).data)) + return targets + + def _preflight(self) -> None: + """Fail fast as :class:`Blocked` when the env cannot be used at all. + + Catches the cases that would otherwise masquerade as dozens of assertion + failures: no kubectl, a missing/unreadable kubeconfig, an unreachable + cluster. Such a run is recorded as Blocked, never as a test failure. + """ + if self.dry_run: + return + if shutil.which("kubectl") is None: + raise Blocked("前置检查失败：PATH 中找不到 kubectl") + problems: list[str] = [] + seen: set[str] = set() + for label, kube in self._probe_targets(): + kubeconfig = kube.kubeconfig or "" + if kubeconfig in seen: + continue + seen.add(kubeconfig) + if kubeconfig and not Path(kubeconfig).is_file(): + problems.append(f"{label} kubeconfig 不可读（{kubeconfig}）") + continue + ran = kube.probe() + if not ran.ok: + lines = (ran.err or ran.out or "").strip().splitlines() + detail = lines[0] if lines else f"rc={ran.rc}" + problems.append(f"{label} 不可达（{detail}）") + if problems: + raise Blocked("前置检查失败：" + "；".join(problems)) + + def _block_run(self, reason: str) -> RunReport: + """Record a setup/preflight failure as Blocked across the whole matrix.""" + self.blocked_reason = reason + self._log(f"run blocked: {reason}") + for case in self.live.values(): + case.status = Status.BLOCKED + case.actual = reason + return self._write_live(finished=utcnow(), environment={}) + + + def _run_flow(self, run: FlowRun, group_scope: _Scope) -> None: + self._log(f"--- {run.kind} {run.group_key}/{run.name} [{', '.join(it.id for it in run.items)}] ---") + self.ctx.begin_flow(run.items) + flow_scope = _Scope() + try: + kwargs = self._resolve_kwargs(run, group_scope, flow_scope) + if run.kind == "case": + item = run.items[0] + with self.ctx.observe(item.id) as obs: + result = run.fn(self.ctx, **kwargs) + if isinstance(result, str) and result: + obs.actual = result + else: + run.fn(self.ctx, **kwargs) + except Blocked as exc: + self.ctx.fail_remaining(str(exc)) + except Restricted as exc: + self.ctx.fail_remaining(str(exc), Status.RESTRICTED) + except Exception as exc: # noqa: BLE001 - abort the flow, block the rest + self._log(traceback.format_exc()) + self.ctx.fail_remaining(f"{type(exc).__name__}: {exc}") + finally: + flow_scope.close(self._log) + self.ctx.finish_flow() + + def _refresh_item(self, item_id: str) -> None: + obs = self.ctx.results.get(item_id) + case = self.live.get(item_id) + if obs is None or case is None: + return + case.status = obs.status + case.actual = obs.actual + case.duration_s = round(obs.duration_s, 1) + for ev in obs.evidence: + if ev not in case.evidence: + case.evidence.append(ev) + self._log(f"[{case.category}] {item_id} {obs.status.value} — {obs.actual}") + self._write_live(finished="") + + # --------------------------------------------------------- fixtures + def _resolve_kwargs(self, run: FlowRun, group_scope: _Scope, flow_scope: _Scope) -> dict[str, Any]: + kwargs: dict[str, Any] = {} + for name in registry.case_signature(run.fn): + if name in run.values: + kwargs[name] = run.values[name] + else: + kwargs[name] = self._fixture_value(name, group_scope, flow_scope) + return kwargs + + def _fixture_value(self, name: str, group_scope: _Scope, flow_scope: _Scope) -> Any: + definition = FIXTURES.get(name) + if definition is None: + raise Blocked(f"未注册 fixture: {name}") + holder = group_scope if definition.scope == "group" else flow_scope + if name in holder.cache: + return holder.cache[name] + if name in holder.errors: + raise holder.errors[name] + kwargs: dict[str, Any] = {} + try: + for dep in registry.case_signature(definition.fn): + kwargs[dep] = self._fixture_value(dep, group_scope, flow_scope) + produced = definition.fn(self.ctx, **kwargs) + if inspect.isgenerator(produced): + value = next(produced) + holder.teardowns.append(produced) + else: + value = produced + except BaseException as exc: # noqa: BLE001 - remembered for dependent flows + holder.errors[name] = exc + raise + holder.cache[name] = value + return value + + # ------------------------------------------------------------- output + def _write_live(self, *, finished: str, environment: dict | None = None) -> RunReport: + report = RunReport( + theme=self.config.theme, + run_id=self.config.run_id, + base_ref=self.config.baseline_ref, + target_ref=self.config.target_ref, + started=self.started, + finished=finished, + config=self.config.raw, + evidence_dir=str(self.config.evidence_dir), + cases=list(self.live.values()), + environment=environment or {}, + blocked_reason=self.blocked_reason, + notes=self.config.get("notes") or {}, + ) + (self.report_dir / "run-manifest.json").write_text( + json.dumps(report.to_dict(), indent=2, ensure_ascii=False) + ) + if not self.dry_run: + write_report(report, self.config) + return report + + def _environment(self) -> dict[str, str]: + env: dict[str, str] = {} + agents = [] + for key in self.ctx.data_keys(): + cl = self.ctx.cl(key) + image = cl.data.jp("deploy", "rlark-agent", "{.spec.template.spec.containers[0].image}") + ready = cl.data.jp("deploy", "rlark-agent", "{.status.readyReplicas}/{.status.replicas}") + agents.append(f"{cl.name} {image} ready {ready}") + env["Cluster Agent 镜像"] = "；".join(agents) + nodes = [] + for key in self.ctx.data_keys(): + lines = [ln for ln in self.ctx.data(key)("get", "nodes", "--no-headers").stdout.splitlines() if ln.strip()] + nodes.append(f"{self.ctx.cl(key).name} {len(lines)} 节点") + env["数据面节点"] = "；".join(nodes) + env["控制组件"] = self.ctx.control.get("deploy").stdout.replace("\n", "；") + if self.ctx.versions.enabled: + env["版本配置"] = ( + f"initial={self.ctx.versions.initial}；" + + "、".join(self.ctx.versions.names()) + ) + current = self.ctx.current_version() + if current: + env["当前版本"] = current + return env + + # ------------------------------------------------------------ archive + def _archive_previous(self) -> None: + manifest = self.report_dir / "run-manifest.json" + if not manifest.exists(): + return + try: + prev = json.loads(manifest.read_text()) + except (OSError, json.JSONDecodeError): + return + prev_target = prev.get("target_ref") + if not prev_target or prev_target == self.config.target_ref: + return + old = str(prev_target).split("@")[-1].split("/")[-1][:12] + report = self.report_dir / self.config.report_file + if report.exists(): + report.rename(report.with_name(f"{report.stem}-{old}{report.suffix}")) + evidence = self.report_dir / "evidence" + if evidence.exists(): + evidence.rename(self._unique_archive(self.report_dir / f"evidence-{old}")) + manifest.rename(self.report_dir / f"run-manifest-{old}.json") + self._log(f"archived previous run ({prev_target}) as *-{old}") + + @staticmethod + def _unique_archive(path: Path) -> Path: + """Keep an existing archive: fall back to ``-1``, ``-2``, ...""" + if not path.exists(): + return path + index = 1 + while True: + candidate = path.with_name(f"{path.name}-{index}") + if not candidate.exists(): + return candidate + index += 1 + + # ------------------------------------------------------------ logging + def _log(self, message: str) -> None: + self._log_handle.write(message + "\n") + self._log_handle.flush() + print(message, file=sys.stderr, flush=True) diff --git a/test/kindtest/kindtest/shell.py b/test/kindtest/kindtest/shell.py new file mode 100644 index 0000000..e7d2075 --- /dev/null +++ b/test/kindtest/kindtest/shell.py @@ -0,0 +1,126 @@ +"""Command execution with logging and dry-run support. + +Every external effect in a test goes through :class:`Shell`, so a whole run can +be inspected with ``--dry-run`` and every command is captured in a per-run log. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +from dataclasses import dataclass +from typing import Callable, Mapping, Sequence + + +@dataclass +class Ran: + argv: list[str] + rc: int + stdout: str = "" + stderr: str = "" + seconds: float = 0.0 + + @property + def ok(self) -> bool: + return self.rc == 0 + + @property + def out(self) -> str: + return self.stdout.strip() + + @property + def err(self) -> str: + return self.stderr.strip() + + def combined(self) -> str: + return (self.stdout + self.stderr).strip() + + +class Shell: + def __init__( + self, + *, + dry_run: bool = False, + verbose: bool = True, + log: Callable[[str], None] | None = None, + cwd: str | None = None, + env: Mapping[str, str] | None = None, + ) -> None: + self.dry_run = dry_run + self.verbose = verbose + self.cwd = cwd + self.env = dict(env or {}) + self._log = log + self.history: list[Ran] = [] + + def _emit(self, message: str) -> None: + if self._log is not None: + self._log(message) + elif self.verbose: + print(message, file=sys.stderr, flush=True) + + def run( + self, + argv: Sequence[object], + *, + cwd: str | None = None, + env: Mapping[str, str] | None = None, + timeout: float | None = None, + input: str | None = None, # noqa: A002 - mirrors subprocess + check: bool = False, + quiet: bool = False, + ) -> Ran: + import time + + argv = [str(a) for a in argv] + cmd = shlex.join(argv) + if self.dry_run: + self._emit(f"[dry-run] {cmd}") + return Ran(argv, 0) + if not quiet: + self._emit(f"$ {cmd}") + full_env = dict(os.environ) + full_env.update(self.env) + if env: + full_env.update(env) + workdir = cwd or self.cwd + started = time.time() + try: + proc = subprocess.run( + argv, + cwd=workdir, + env=full_env, + input=input, + capture_output=True, + text=True, + timeout=timeout, + ) + except FileNotFoundError as exc: + ran = Ran(argv, 127, "", str(exc), time.time() - started) + self.history.append(ran) + if check: + raise RuntimeError(f"command not found: {cmd}") from exc + return ran + except subprocess.TimeoutExpired as exc: + stdout = _text(exc.stdout) + stderr = _text(exc.stderr) or f"timeout after {timeout}s" + ran = Ran(argv, 124, stdout, stderr, time.time() - started) + self.history.append(ran) + if check: + raise RuntimeError(f"timeout after {timeout}s: {cmd}") + return ran + ran = Ran(argv, proc.returncode, proc.stdout or "", proc.stderr or "", time.time() - started) + self.history.append(ran) + if check and ran.rc != 0: + raise RuntimeError(f"command failed rc={ran.rc}: {cmd}\n{ran.err}") + return ran + + +def _text(value: object) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode(errors="replace") + return str(value) diff --git a/test/kindtest/kindtest/versions.py b/test/kindtest/kindtest/versions.py new file mode 100644 index 0000000..faad36c --- /dev/null +++ b/test/kindtest/kindtest/versions.py @@ -0,0 +1,435 @@ +"""Named versions and rollout deployment. + +A config may declare several named *versions*. Each version is a git ``ref`` +plus one or more **named images** (``rlark``, ``ui``, ...), because different +components ship different images. The ``rollout`` block lists the workloads to +update and, per workload, which image slot of the version it should run. + +The framework deploys the ``initial`` version before any flow runs and exposes +``t.upgrade(name)`` so a test can switch to any other declared version at a +controlled point in a flow. + +Deployment is idempotent: when every live container already carries the +requested image (and ``--image=`` style argument), :meth:`VersionDeployer.deploy` +patches nothing and only verifies rollout status. Repeating an upgrade, or +deploying the version that is already running, is therefore cheap and safe. + +Config shape:: + + versions: + initial: target + definitions: + base: + ref: main@aaaaaaa + images: {rlark: rlark:baseline-aaaaaaa, ui: rlark-ui:baseline} + target: + ref: main@bbbbbbb + images: {rlark: rlark:target-bbbbbbb, ui: rlark-ui:target} + rollout: + control: # applied in the control cluster + - {kind: deploy, name: rlark-controller-manager, container: rlark-controller-manager, slot: rlark} + - {kind: deploy, name: rlark-ui, container: rlark-ui, slot: ui} + data: # applied in every data cluster + - {kind: deploy, name: rlark-agent, container: rlark-agent, slot: rlark, args: ["--image="]} + - {kind: ds, name: rlark-agent-node, container: rlark-agent-node, slot: rlark} + +A single-image version may be written as ``image: `` (shorthand for a +``default`` slot). A rollout target without ``slot`` uses the version's +``default`` slot, or its only image; a version with several slots and no +``default`` must be addressed by slot. ``kind`` accepts ``deploy``/``ds``/``sts``. +``args`` entries are argument prefixes whose value is rewritten to the selected +image, e.g. ``--image=`` turns ``--image=rlark:old`` into ``--image=rlark:new``. +A prefix that is absent is an error, because silently appending a new argument +would run the workload with a configuration the environment never had; set +``appendArgs: true`` on the target to opt into appending instead. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable + +from .kube import Cluster, Kube + +_KINDS = { + "deploy": "deploy", + "deployment": "deploy", + "ds": "ds", + "daemonset": "ds", + "sts": "sts", + "statefulset": "sts", +} + + +class VersionError(RuntimeError): + """An undefined version, an unusable rollout config, or a failed rollout.""" + + +def resolve_image(images: dict[str, str], slot: str = "", *, what: str = "版本") -> str: + """The single image for ``slot``, or the ``default``/only slot. + + Raises :class:`VersionError` when a named ``slot`` is missing, or when the + version defines several slots without a ``default`` and no slot was given - + callers never silently pick an image the config did not intend. For a + human-readable summary use :func:`display_image`. + """ + if slot: + if slot in images: + return images[slot] + available = "、".join(images) or "（无）" + raise VersionError(f"{what} 未定义镜像 slot {slot!r}；可用：{available}") + if "default" in images: + return images["default"] + if len(images) == 1: + return next(iter(images.values())) + available = "、".join(images) or "（无）" + raise VersionError(f"{what} 定义了多个镜像 slot，必须指定 slot；可用：{available}") + + +def display_image(images: dict[str, str]) -> str: + """A human-readable image for a version, for reports and logs. + + Prefers the ``default`` slot or the only image. A version that defines + several slots without a ``default`` is listed in full (``rlark=…、ui=…``) + rather than silently collapsed to one of them. Use :func:`resolve_image` + (or :meth:`Version.image_for`) when a single concrete image is required. + """ + if not images: + return "" + if "default" in images: + return images["default"] + if len(images) == 1: + return next(iter(images.values())) + return "、".join(f"{slot}={image}" for slot, image in images.items()) + + +@dataclass +class Version: + name: str + ref: str = "" + images: dict[str, str] = field(default_factory=dict) + + @classmethod + def parse(cls, name: str, raw: Any) -> "Version": + raw = raw or {} + if not isinstance(raw, dict): + raise VersionError(f"versions.definitions.{name} must be a mapping") + images: dict[str, str] = {} + if isinstance(raw.get("images"), dict): + images = {str(k): str(v) for k, v in raw["images"].items()} + elif raw.get("image"): + images = {"default": str(raw["image"])} + if not images: + raise VersionError(f"版本 {name!r} 未定义任何镜像（images 或 image）") + return cls(name=str(name), ref=str(raw.get("ref", "")), images=images) + + @property + def image(self) -> str: + """Human-readable image(s) for display; see :func:`display_image`.""" + return display_image(self.images) + + def image_for(self, slot: str) -> str: + """The single image for ``slot`` (or the default/only slot). + + Raises :class:`VersionError` when the version defines several slots and + neither ``slot`` nor a ``default`` names one, so a rollout can never pick + an image the config did not intend. + """ + return resolve_image(self.images, slot, what=f"版本 {self.name!r}") + + +@dataclass +class RolloutTarget: + kind: str + name: str + container: str + slot: str = "" + args: list[str] = field(default_factory=list) + append_missing: bool = False + + @classmethod + def parse(cls, raw: dict[str, Any]) -> "RolloutTarget": + if not isinstance(raw, dict): + raise VersionError(f"rollout target must be a mapping: {raw!r}") + kind = str(raw.get("kind", "deploy")).lower() + if kind not in _KINDS: + raise VersionError(f"unknown rollout kind {kind!r}") + name = str(raw.get("name", "")) + if not name: + raise VersionError("rollout target requires 'name'") + return cls( + kind=_KINDS[kind], + name=name, + container=str(raw.get("container", name)), + slot=str(raw.get("slot", "")), + args=[str(a) for a in (raw.get("args") or [])], + append_missing=bool(raw.get("appendArgs", False)), + ) + + @property + def resource(self) -> str: + return f"{self.kind}/{self.name}" + + +@dataclass +class DeployResult: + name: str + images: dict[str, str] + changed: bool + targets: int = 0 + + +class VersionDeployer: + """Resolves named versions and rolls the configured workloads to one of them.""" + + def __init__( + self, + config: Any, + *, + control: Kube, + clusters: dict[str, Cluster], + log: Callable[[str], None] | None = None, + ) -> None: + self.config = config + self.control = control + self.clusters = clusters + self._log = log or (lambda _m: None) + + raw_defs = config.get("versions.definitions") or {} + if not isinstance(raw_defs, dict): + raise VersionError("versions.definitions must be a mapping") + self.definitions: dict[str, Version] = { + str(name): Version.parse(str(name), entry) for name, entry in raw_defs.items() + } + + self.initial = str(config.get("versions.initial", "")) + if self.initial and self.initial not in self.definitions: + raise VersionError( + f"versions.initial={self.initial!r} 未在 versions.definitions 中定义" + ) + if not self.initial and self.definitions: + self.initial = next(iter(self.definitions)) + + self.control_targets = [ + RolloutTarget.parse(x) for x in (config.get("versions.rollout.control") or []) + ] + self.data_targets = [ + RolloutTarget.parse(x) for x in (config.get("versions.rollout.data") or []) + ] + self.current_name = "" + + # ------------------------------------------------------------- identity + @property + def enabled(self) -> bool: + return bool(self.definitions) + + @property + def rollout_configured(self) -> bool: + return bool(self.control_targets or self.data_targets) + + def names(self) -> list[str]: + return list(self.definitions) + + def version(self, name: str) -> Version: + try: + return self.definitions[name] + except KeyError: + available = "、".join(self.definitions) or "（无）" + raise VersionError(f"未定义版本 {name!r}；可用版本：{available}") + + def image(self, name: str, slot: str = "") -> str: + """The concrete image for ``name`` (and optional ``slot``). + + Ambiguous versions raise rather than silently picking a slot; for a + human-readable summary use ``version(name).image`` (or + ``versions.version(name).images``). + """ + return self.version(name).image_for(slot) + + def ref(self, name: str) -> str: + return self.version(name).ref + + # ------------------------------------------------------------ deployment + def deploy(self, name: str, *, timeout: int = 300, wait: bool = True) -> DeployResult: + version = self.version(name) + if not self.rollout_configured: + raise VersionError("未配置 versions.rollout，无法部署/升级版本") + + changed = False + targets = 0 + for target in self.control_targets: + targets += 1 + changed |= self._apply( + self.control, self.config.control_namespace, target, version.image_for(target.slot) + ) + for cluster in self.clusters.values(): + for target in self.data_targets: + targets += 1 + changed |= self._apply( + cluster.data, cluster.data_ns, target, version.image_for(target.slot) + ) + + if wait: + for target in self.control_targets: + self._await(self.control, self.config.control_namespace, target, timeout) + for cluster in self.clusters.values(): + for target in self.data_targets: + self._await(cluster.data, cluster.data_ns, target, timeout) + + self.current_name = name + self._log( + f"version {name} deployed (images={version.images}, changed={changed}, targets={targets})" + ) + return DeployResult(name=name, images=dict(version.images), changed=changed, targets=targets) + + def detect(self) -> str: + """Best-effort: the declared version whose image matches the live target.""" + probe = self._probe() + if probe is None: + return "" + kube, ns, target = probe + live = self._image_of(kube, ns, target) + if not live: + return "" + for version in self.definitions.values(): + try: + if version.image_for(target.slot) == live: + return version.name + except VersionError: + continue + return "" + + def audit(self, name: str) -> list[dict[str, Any]]: + """Every rollout target with its expected and live image for ``name``.""" + version = self.version(name) + rows: list[dict[str, Any]] = [] + for target in self.control_targets: + rows.append( + self._audit_one(self.control, self.config.control_namespace, "control", target, version) + ) + for key, cluster in self.clusters.items(): + for target in self.data_targets: + rows.append(self._audit_one(cluster.data, cluster.data_ns, key, target, version)) + return rows + + def _audit_one(self, kube, ns, scope, target, version) -> dict[str, Any]: + expected = version.image_for(target.slot) + try: + live = self._image_of(kube, ns, target) + except Exception: # noqa: BLE001 - a missing target is just not ok + live = "" + return { + "scope": scope, + "resource": target.resource, + "container": target.container, + "slot": target.slot, + "expected": expected, + "live": live, + "ok": live == expected, + } + + # --------------------------------------------------------------- internal + def _probe(self) -> tuple[Kube, str, RolloutTarget] | None: + if self.control_targets: + return self.control, self.config.control_namespace, self.control_targets[0] + if self.data_targets and self.clusters: + cluster = next(iter(self.clusters.values())) + return cluster.data, cluster.data_ns, self.data_targets[0] + return None + + def _image_of(self, kube: Kube, ns: str, target: RolloutTarget) -> str: + obj = kube.json(target.kind, target.name, ns=ns) + for container in _containers(obj): + if container.get("name") == target.container: + return str(container.get("image", "")) + return "" + + def _apply(self, kube: Kube, ns: str, target: RolloutTarget, image: str) -> bool: + obj = kube.json(target.kind, target.name, ns=ns) + containers = _containers(obj) + index = next((i for i, c in enumerate(containers) if c.get("name") == target.container), None) + if index is None: + raise VersionError(f"{target.resource} 中不存在容器 {target.container}") + container = containers[index] + path = f"/spec/template/spec/containers/{index}" + + # One JSON patch for image *and* args: a single spec write means a single + # rollout, with no intermediate "new image, old args" state. + ops: list[dict[str, Any]] = [] + if container.get("image") != image: + ops.append({"op": "replace", "path": f"{path}/image", "value": image}) + if target.args: + new_args = _rewrite_args( + container.get("args") or [], + target.args, + image, + append_missing=target.append_missing, + where=target.resource, + ) + if new_args is not None: + # ``replace`` needs an existing member; ``add`` creates ``args``. + op = "replace" if "args" in container else "add" + ops.append({"op": op, "path": f"{path}/args", "value": new_args}) + + if not ops: + return False + ran = kube.patch(target.kind, target.name, ops, ns=ns, type="json") + if not ran.ok: + raise VersionError(f"{target.resource} patch 失败: {ran.err.strip()[:200]}") + return True + + def _await(self, kube: Kube, ns: str, target: RolloutTarget, timeout: int) -> None: + ran = kube.rollout_status(target.resource, ns=ns, timeout=timeout) + if not ran.ok: + raise VersionError( + f"{target.resource} rollout 未在 {timeout}s 内完成: {ran.err.strip()[:200]}" + ) + + +def _containers(obj: Any) -> list[dict]: + spec = ((obj or {}).get("spec") or {}).get("template") or {} + return list((spec.get("spec") or {}).get("containers") or []) + + +def _rewrite_args( + args: list[str], + prefixes: list[str], + image: str, + *, + append_missing: bool = False, + where: str = "", +) -> list[str] | None: + """Return rewritten args, or ``None`` when nothing needs changing. + + A prefix that is not present rewrites nothing; that is an authoring/config + error (the workload would keep running with a configuration the run did not + intend) unless ``append_missing`` opted into appending the argument. + """ + out = list(args) + changed = False + for prefix in prefixes: + desired = prefix + image if prefix.endswith("=") else f"{prefix}={image}" + found = False + for i, arg in enumerate(out): + if _matches_prefix(arg, prefix): + found = True + if arg != desired: + out[i] = desired + changed = True + break + if not found: + if not append_missing: + location = f"{where} " if where else "" + raise VersionError( + f"{location}启动参数中未找到 {prefix!r}；框架默认只改写已存在的参数，" + f"如需缺失时追加请设置 appendArgs: true" + ) + out.append(desired) + changed = True + return out if changed else None + + +def _matches_prefix(arg: str, prefix: str) -> bool: + if prefix.endswith(("=", " ")): + return arg.startswith(prefix) + return arg == prefix or arg.startswith(prefix + "=") or arg.startswith(prefix + " ") diff --git a/test/kindtest/kt b/test/kindtest/kt new file mode 100755 index 0000000..1a878f3 --- /dev/null +++ b/test/kindtest/kt @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# kindtest 启动器：让任意工作目录都能运行框架，无需安装。 +# +# 用法（在你自己写测例的目录里按路径调用本脚本）： +# /test/kindtest/kt plan -c config.yaml +# /test/kindtest/kt run -c config.yaml +set -euo pipefail + +# 本脚本位于 /kt，框架根目录即其所在目录。 +self="$(readlink -f "${BASH_SOURCE[0]}")" +framework_dir="$(cd "$(dirname "$self")" && pwd)" + +export PYTHONPATH="$framework_dir${PYTHONPATH:+:$PYTHONPATH}" +exec python3 -m kindtest "$@" diff --git a/test/kindtest/test_framework.py b/test/kindtest/test_framework.py new file mode 100644 index 0000000..2710dc0 --- /dev/null +++ b/test/kindtest/test_framework.py @@ -0,0 +1,404 @@ +"""Self-tests for the kindtest framework itself. + +No cluster is required: these exercise config parsing, the evidence book, the +run-archive step and the setup preflight (which fails as Blocked, not FAIL). +Run from ``test/kindtest``: + + python3 test_framework.py + python3 -m unittest test_framework -v +""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from kindtest.config import Config # noqa: E402 +from kindtest.context import EvidenceBook # noqa: E402 +from kindtest.model import Status # noqa: E402 +from kindtest.registry import Blocked, FlowRun, Item, RegistryError # noqa: E402 +from kindtest.runner import Runner # noqa: E402 +from kindtest.shell import Ran # noqa: E402 +from kindtest.versions import ( # noqa: E402 + RolloutTarget, + Version, + VersionDeployer, + VersionError, + _rewrite_args, +) + +_TESTS_DIR = Path(__file__).resolve().parent / "tests" + +_CONFIG = """\ +run: + id: selftest +clusters: + control: + kubeconfig: {base}/kubeconfig + kcp: {base}/kcp.kubeconfig + data: + a: + name: data-a + clusterId: rlark-data-a + kubeconfig: {base}/a.kubeconfig +report: + dir: {base}/reports +tests: + dir: {tests} +""" + + +def _runner(base: Path, *, extra: str = "", dry_run: bool = True) -> Runner: + cfg_path = base / "config.yaml" + cfg_path.write_text(_CONFIG.format(base=base, tests=_TESTS_DIR) + extra) + return Runner(Config.load(cfg_path), dry_run=dry_run) + + +class EvidenceBookTest(unittest.TestCase): + def test_add_creates_directory_on_demand(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "evidence" + book = EvidenceBook(root) + # nothing is created before the run archives a previous directory + self.assertFalse(root.exists()) + book.add("a.txt", "x") + self.assertTrue((root / "a.txt").is_file()) + + def test_add_self_heals_after_directory_moves(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "evidence" + book = EvidenceBook(root) + book.add("a.txt", "x") + # _archive_previous() renames evidence/ -> evidence-/ + root.rename(root.with_name("evidence-old")) + book.add("b.txt", "y") # must not raise FileNotFoundError + self.assertTrue((root / "b.txt").is_file()) + + +class RunnerArchiveTest(unittest.TestCase): + def _seed_previous_run(self, runner: Runner) -> Path: + rd = runner.report_dir + (rd / "run-manifest.json").write_text(json.dumps({"target_ref": "main@470a3d6c"})) + (rd / "full-test-result.md").write_text("# old\n") + (rd / "evidence").mkdir() + (rd / "evidence" / "old.txt").write_text("old\n") + return rd + + def test_constructor_does_not_archive(self) -> None: + """``kt plan`` only constructs a Runner; it must not touch the report dir.""" + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + (base / "reports").mkdir() + (base / "reports" / "run-manifest.json").write_text( + json.dumps({"target_ref": "main@470a3d6c"}) + ) + _runner(base) + self.assertTrue((base / "reports" / "run-manifest.json").is_file()) + + def test_archive_then_evidence_write_recovers(self) -> None: + """Regression: after archiving evidence/, the next add() must still work.""" + with tempfile.TemporaryDirectory() as tmp: + runner = _runner(Path(tmp)) + # construction is lazy: no evidence/ exists before archiving + self.assertFalse(runner.config.evidence_dir.exists()) + rd = self._seed_previous_run(runner) + + runner._archive_previous() + runner.evidence.ensure() + + self.assertTrue((rd / "run-manifest-470a3d6c.json").is_file()) + self.assertTrue((rd / "full-test-result-470a3d6c.md").is_file()) + self.assertTrue((rd / "evidence-470a3d6c" / "old.txt").is_file()) + + runner.evidence.add("new.txt", "new") + self.assertTrue((rd / "evidence" / "new.txt").is_file()) + + def test_archive_survives_alternating_targets(self) -> None: + """Repeated upgrade checks that flip the target must not collide on archive.""" + with tempfile.TemporaryDirectory() as tmp: + runner = _runner(Path(tmp)) + rd = runner.report_dir + for i, target in enumerate(["main@sha1", "main@sha2", "main@sha1", "main@sha2"]): + (rd / "run-manifest.json").write_text(json.dumps({"target_ref": target})) + (rd / "evidence").mkdir(exist_ok=True) + (rd / "evidence" / f"e{i}.txt").write_text(target) + runner._archive_previous() # must not raise OSError + self.assertTrue((rd / "evidence-sha1").is_dir()) + self.assertTrue((rd / "evidence-sha2").is_dir()) + self.assertTrue((rd / "evidence-sha1-1").is_dir()) + self.assertTrue((rd / "evidence-sha2-1").is_dir()) + + +class PreflightTest(unittest.TestCase): + """Setup/preflight problems must be Blocked, never a pile of test failures.""" + + def test_missing_kubeconfig_blocks_every_item(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + runner = _runner(Path(tmp), dry_run=False) + runner.load_tests() + report = runner.run() + + self.assertTrue(report.blocked_reason.startswith("前置检查失败"), report.blocked_reason) + self.assertTrue(report.cases) + self.assertTrue(all(c.status is Status.BLOCKED for c in report.cases)) + self.assertFalse([c for c in report.cases if c.status is Status.FAIL]) + markdown = (runner.report_dir / runner.config.report_file).read_text() + self.assertIn("运行未开始", markdown) + manifest = json.loads((runner.report_dir / "run-manifest.json").read_text()) + self.assertEqual(manifest["blocked_reason"], report.blocked_reason) + + def test_kubectl_missing_blocks(self) -> None: + from unittest import mock + + with tempfile.TemporaryDirectory() as tmp: + runner = _runner(Path(tmp), dry_run=False) + with mock.patch("kindtest.runner.shutil.which", return_value=None): + with self.assertRaises(Blocked) as ctx: + runner._preflight() + self.assertIn("kubectl", str(ctx.exception)) + + def test_initial_version_failure_blocks(self) -> None: + from unittest import mock + + with tempfile.TemporaryDirectory() as tmp: + runner = _runner(Path(tmp), dry_run=False, extra=""" +versions: + initial: target + definitions: + target: {ref: t, images: {rlark: rlark:target}} + rollout: + control: + - {kind: deploy, name: rlark-gateway, container: rlark-gateway, slot: rlark} +""") + with mock.patch.object(runner.ctx, "deploy_version", side_effect=RuntimeError("boom")): + with self.assertRaises(Blocked) as ctx: + runner._deploy_initial_version() + self.assertIn("初始版本", str(ctx.exception)) + + +class UniqueItemIdTest(unittest.TestCase): + """Item ids key the report matrix; a duplicate must be rejected, not merged.""" + + @staticmethod + def _runs(*ids: str) -> list[FlowRun]: + return [ + FlowRun("REG", "REG", f"flow-{item_id}", "flow", lambda t: None, [Item(item_id, item_id)]) + for item_id in ids + ] + + def test_duplicate_ids_are_rejected(self) -> None: + with self.assertRaises(RegistryError) as ctx: + Runner._assert_unique_item_ids(self._runs("REG-01", "REG-01")) + msg = str(ctx.exception) + self.assertIn("REG-01", msg) + self.assertIn("flow-REG-01", msg) # both declaring flows are listed + + def test_unique_ids_pass(self) -> None: + Runner._assert_unique_item_ids(self._runs("REG-01", "REG-02")) # must not raise + + +class OnlySelectionTest(unittest.TestCase): + """``--only`` is exact by default; a trailing ``*`` opts into prefix matching.""" + + def _runner(self, tmp: str) -> Runner: + return _runner(Path(tmp)) + + def test_exact_id_matches_base_and_parametrized(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + runner = self._runner(tmp) + runner.only = ["REG-01"] + self.assertTrue(runner._item_selected("REG-01")) + self.assertTrue(runner._item_selected("REG-01[data-a]")) + self.assertFalse(runner._item_selected("REG-011")) + self.assertFalse(runner._item_selected("REG-010")) + + def test_trailing_star_selects_a_prefix(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + runner = self._runner(tmp) + runner.only = ["REG-0*"] + self.assertTrue(runner._item_selected("REG-01")) + self.assertTrue(runner._item_selected("REG-09[data-a]")) + self.assertFalse(runner._item_selected("REG-10")) + + +class RewriteArgsTest(unittest.TestCase): + """``_rewrite_args`` rewrites in place and errors on a missing prefix.""" + + def test_rewrites_in_place(self) -> None: + self.assertEqual( + ["--image=rlark:new", "--mode=cluster"], + _rewrite_args(["--image=rlark:old", "--mode=cluster"], ["--image="], "rlark:new"), + ) + + def test_noop_returns_none(self) -> None: + self.assertIsNone(_rewrite_args(["--image=rlark:new"], ["--image="], "rlark:new")) + + def test_missing_prefix_raises_by_default(self) -> None: + with self.assertRaises(VersionError) as ctx: + _rewrite_args(["--mode=cluster"], ["--image="], "rlark:new") + self.assertIn("--image=", str(ctx.exception)) + + def test_append_missing_is_explicit_opt_in(self) -> None: + self.assertEqual( + ["--mode=cluster", "--image=rlark:new"], + _rewrite_args(["--mode=cluster"], ["--image="], "rlark:new", append_missing=True), + ) + + +class _RecordingKube: + """Kube stand-in for ``_apply``: records patches and bare ``kubectl`` calls.""" + + def __init__(self, obj: dict) -> None: + self._obj = obj + self.patches: list[tuple[str, str, list]] = [] + self.verbs: list[list] = [] + + def json(self, kind: str, name: str, *, ns: str | None = None) -> dict: + return self._obj + + def patch(self, kind: str, name: str, patch: list, *, ns: str | None = None, + type: str | None = None) -> Ran: # noqa: A002 - mirrors Kube.patch + self.patches.append((kind, name, patch)) + return Ran(argv=[], rc=0) + + def __call__(self, *argv: object) -> Ran: + self.verbs.append(list(argv)) + return Ran(argv=[], rc=0) + + +class ApplyPatchTest(unittest.TestCase): + """Issue: image and args must land in ONE spec write (one rollout, no halfway state).""" + + def _apply(self, obj: dict, target: RolloutTarget, image: str) -> tuple[_RecordingKube, bool]: + kube = _RecordingKube(obj) + changed = VersionDeployer._apply(None, kube, "rlark-system", target, image) # type: ignore[arg-type] + return kube, changed + + def test_image_and_args_patched_once(self) -> None: + obj = {"spec": {"template": {"spec": {"containers": [ + {"name": "rlark-agent", "image": "rlark:old", + "args": ["--mode=cluster", "--image=rlark:old"]}, + ]}}}} + target = RolloutTarget(kind="deploy", name="rlark-agent", container="rlark-agent", + slot="rlark", args=["--image="]) + kube, changed = self._apply(obj, target, "rlark:new") + self.assertTrue(changed) + self.assertEqual([], kube.verbs) # no separate `kubectl set image` + self.assertEqual(1, len(kube.patches)) # exactly one spec write + ops = kube.patches[0][2] + self.assertEqual( + ["/spec/template/spec/containers/0/image", + "/spec/template/spec/containers/0/args"], + [op["path"] for op in ops], + ) + self.assertEqual("rlark:new", ops[0]["value"]) + self.assertEqual(["--mode=cluster", "--image=rlark:new"], ops[1]["value"]) + + def test_absent_args_are_added(self) -> None: + obj = {"spec": {"template": {"spec": {"containers": [{"name": "c", "image": "old"}]}}}} + target = RolloutTarget(kind="deploy", name="d", container="c", args=["--image="], + append_missing=True) + kube, changed = self._apply(obj, target, "new") + self.assertTrue(changed) + op = kube.patches[0][2][-1] + self.assertEqual("add", op["op"]) + self.assertEqual(["--image=new"], op["value"]) + + def test_no_change_patches_nothing(self) -> None: + obj = {"spec": {"template": {"spec": {"containers": [ + {"name": "c", "image": "new", "args": ["--image=new"]}, + ]}}}} + target = RolloutTarget(kind="deploy", name="d", container="c", args=["--image="]) + kube, changed = self._apply(obj, target, "new") + self.assertFalse(changed) + self.assertEqual([], kube.patches) + + def test_missing_prefix_fails_the_rollout(self) -> None: + obj = {"spec": {"template": {"spec": {"containers": [ + {"name": "c", "image": "old", "args": ["--mode=cluster"]}, + ]}}}} + target = RolloutTarget(kind="deploy", name="d", container="c", args=["--image="]) + with self.assertRaises(VersionError) as ctx: + self._apply(obj, target, "new") + self.assertIn("--image=", str(ctx.exception)) + + +class SlotSemanticsTest(unittest.TestCase): + """Slots are resolved one way: display lists all, selection is strict.""" + + def test_display_lists_every_slot(self) -> None: + version = Version(name="target", images={"rlark": "rlark:x", "ui": "ui:x"}) + self.assertEqual("rlark=rlark:x、ui=ui:x", version.image) + + def test_display_prefers_default_then_single(self) -> None: + self.assertEqual("d", Version(name="t", images={"default": "d", "ui": "u"}).image) + self.assertEqual("only", Version(name="t", images={"rlark": "only"}).image) + + def test_selection_without_slot_is_strict(self) -> None: + version = Version(name="target", images={"rlark": "a", "ui": "b"}) + with self.assertRaises(VersionError): + version.image_for("") + self.assertEqual("b", version.image_for("ui")) + with self.assertRaises(VersionError): + version.image_for("nope") + + def test_config_and_version_agree(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + cfg_path = base / "config.yaml" + cfg_path.write_text( + _CONFIG.format(base=base, tests=_TESTS_DIR) + + "\nversions:\n" + + " initial: target\n" + + " definitions:\n" + + " target: {ref: t, images: {rlark: rlark:x, ui: ui:x}}\n" + ) + config = Config.load(cfg_path) + # display path lists every slot and agrees with Version.image + self.assertEqual( + Version(name="target", images={"rlark": "rlark:x", "ui": "ui:x"}).image, + config.target_image_display, + ) + self.assertEqual("rlark=rlark:x、ui=ui:x", config.target_image_display) + # value path is strict: several slots without a default is an error + with self.assertRaises(VersionError): + _ = config.target_image + + def test_config_single_image_resolves(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + cfg_path = base / "config.yaml" + cfg_path.write_text( + _CONFIG.format(base=base, tests=_TESTS_DIR) + + "\nversions:\n" + + " initial: target\n" + + " definitions:\n" + + " target: {ref: t, image: rlark:only}\n" + ) + config = Config.load(cfg_path) + self.assertEqual("rlark:only", config.target_image) + self.assertEqual("rlark:only", config.target_image_display) + + def test_render_is_lazy_for_ambiguous_target(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + runner = _runner( + Path(tmp), + extra="\nversions:\n" + + " initial: target\n" + + " definitions:\n" + + " target: {ref: t, images: {rlark: rlark:x, ui: ui:x}}\n", + ) + # a template that does not use the ambiguous image still renders + self.assertEqual("hi", runner.ctx.render("hi")) + # using it is an explicit error, not a silent pick + with self.assertRaises(VersionError): + runner.ctx.render("image={{targetImage}}") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/kindtest/tests/00_env.py b/test/kindtest/tests/00_env.py new file mode 100644 index 0000000..83b8049 --- /dev/null +++ b/test/kindtest/tests/00_env.py @@ -0,0 +1,115 @@ +"""ENV: layered environment health - read-only checks on the live clusters. + +This file is a good template: each case is a small function, the parametrized +ones share one body, and every assertion returns an auditable fact. +""" + +from __future__ import annotations + +from kindtest import group + +env = group("ENV", title="环境和组件健康") + + +@env.case("ENV-01", "Kind 节点容器", expect="配置中的集群节点容器全部运行", source="环境") +def kind_containers(t): + names = [str(t.config.get("clusters.control.name"))] + [t.cl(k).name for k in t.data_keys()] + expected = [n for name in names for n in t.kind.nodes(name)] + up = set(t.kind.containers_up()) + missing = [n for n in expected if n not in up] + assert expected, "未发现任何 Kind 节点容器" + assert not missing, f"未运行节点容器: {missing}" + return f"{len(expected)} 个节点容器 Up" + + +@env.case( + "ENV-02", + "Kubernetes 节点 Ready", + expect="各集群节点全部 Ready", + source="环境", + params=[{"cluster": "control"}, {"cluster": "a"}, {"cluster": "b"}], + ids=["control", "a", "b"], +) +def nodes_ready(t, cluster): + kube = t.control if cluster == "control" else t.data(cluster) + lines = [ln for ln in kube("get", "nodes", "--no-headers").stdout.splitlines() if ln.strip()] + ready = [ln for ln in lines if " Ready" in ln] + return t.eq(len(ready), len(lines), f"{cluster} Ready 节点") + f"/{len(lines)}" + + +@env.case("ENV-03", "控制面组件", expect="KCP/PostgreSQL/Server/Gateway/Controller Manager/UI Ready", source="环境") +def control_plane(t): + lines = [ln for ln in t.control("get", "pods", "-n", t.config.control_namespace, + "--no-headers").stdout.splitlines() if ln.strip()] + ready, bad = 0, [] + for line in lines: + parts = line.split() + if len(parts) < 3 or "/" not in parts[1]: + continue + got, want = parts[1].split("/") + if got == want and parts[2] in ("Running", "Completed"): + ready += 1 + else: + bad.append(f"{parts[0]}({parts[2]})") + assert not bad, f"异常组件: {bad}" + assert ready >= 6, f"控制面 Ready Pod 过少: {ready}" + return f"{ready} 个控制面 Pod Running/Completed" + + +@env.case( + "ENV-04", + "数据面 Agent", + expect="两 Cluster Agent 与 Node Agent Ready", + source="环境", + params=[{"cluster": "a"}, {"cluster": "b"}], + ids=["a", "b"], +) +def data_agents(t, cluster): + cl = t.cl(cluster) + dep = cl.data.jp("deploy", "rlark-agent", "{.status.readyReplicas}/{.spec.replicas}") + ds = cl.data.jp("ds", "rlark-agent-node", "{.status.numberReady}/{.status.desiredNumberScheduled}") + _all_ready(dep, "Cluster Agent") + _all_ready(ds, "Node Agent") + return f"deploy={dep} ds={ds}" + + +@env.case("ENV-05", "管理面 API 和 CRD", expect="KCP API 可访问且 rlinf.io CRD 已注册", source="环境") +def management_api(t): + ran = t.succeeded(t.mgmt("api-resources", "--api-group=rlinf.io", "--no-headers"), "api-resources") + names = {ln.split()[0] for ln in ran.stdout.splitlines() if ln.strip()} + required = {"jobs", "tasks", "workflows", "domains", "domainpeers", "nodes", "pods", "addons"} + missing = sorted(required - names) + assert not missing, f"缺少 CRD 资源: {missing}" + return f"{len(names)} 个 rlinf.io 资源可用" + + +@env.case( + "ENV-06", + "默认 StorageClass", + expect="两侧数据集群均有默认 StorageClass", + source="环境", + params=[{"cluster": "a"}, {"cluster": "b"}], + ids=["a", "b"], +) +def default_storageclass(t, cluster): + out = t.data(cluster)("get", "sc", "--no-headers").stdout + defaults = [ln.split()[0] for ln in out.splitlines() if "(default)" in ln] + assert defaults, "未发现默认 StorageClass" + return f"default={','.join(defaults)}" + + +@env.case("ENV-07", "基础镜像预检", expect="配置的离线基础镜像保留在宿主", source="环境") +def base_images(t): + wanted = [str(x) for x in (t.config.get("images.base") or [])] + if not wanted: + t.restrict("未配置 images.base，跳过基础镜像预检") + return "未配置 images.base" + present = [img for img in wanted if t.kind.image_id(img)] + missing = [img for img in wanted if img not in present] + assert not missing, f"缺失基础镜像: {missing}" + return f"{len(present)}/{len(wanted)} 个离线基础镜像保留" + + +def _all_ready(ratio: str, what: str) -> None: + got, _, want = ratio.partition("/") + assert got and want and got == want, f"{what} 未就绪: {ratio or 'n/a'}" diff --git a/test/kindtest/tests/05_mig.py b/test/kindtest/tests/05_mig.py new file mode 100644 index 0000000..073c4c1 --- /dev/null +++ b/test/kindtest/tests/05_mig.py @@ -0,0 +1,58 @@ +"""MIG: 在途升级 - 用配置里定义的命名版本，在流程中途要求框架升级。 + +只有 ``config.yaml`` 定义了 ``versions`` **且** ``versions.rollout`` 时才能 +真正执行；否则整组记为“受限”。框架会在这组流程之前先部署 +``versions.initial``，因此这里只需要 ``t.upgrade(name)`` 在受控时点切换。 + +配置示例见 README“多版本与升级”与 ``config.example.yaml``：: + + versions: + initial: target + definitions: + base: {ref: main@aaaaaaa, images: {rlark: rlark:baseline, ui: rlark-ui:baseline}} + target: {ref: main@bbbbbbb, images: {rlark: rlark:target, ui: rlark-ui:target}} + rollout: + data: + - {kind: deploy, name: rlark-agent, container: rlark-agent, slot: rlark, args: ["--image="]} + - {kind: ds, name: rlark-agent-node, container: rlark-agent-node, slot: rlark} +""" + +from __future__ import annotations + +from kindtest import Restricted, group + +mig = group("MIG", title="在途升级和不变量") + + +@mig.flow( + "命名版本在途升级", + items=[ + ("MIG-01", "初始版本就绪", "运行开始部署的初始版本可被识别且目标 Ready"), + ("MIG-02", "升级到目标版本", "t.upgrade(目标) 后镜像切到目标且 Ready"), + ("MIG-03", "版本可回退", "t.upgrade(初始) 后镜像回到初始"), + ], +) +def named_version_upgrade(t): + if not t.versions.enabled or not t.versions.rollout_configured: + raise Restricted("未配置 versions.rollout，跳过在途升级场景") + + initial = t.versions.initial + target = next((n for n in t.version_names() if n != initial), initial) + + with t.observe("MIG-01"): + current = t.current_version() + t.truthy(current == initial, "初始版本已部署", + f"detected={current!r} initial={initial!r}") + t.evidence(f"detected={current} initial={initial}", "MIG-initial.txt") + + with t.observe("MIG-02"): + result = t.upgrade(target) + t.eq(t.current_version(), target, "升级后版本") + t.evidence( + f"{initial} -> {target} images={result.images} changed={result.changed}", "MIG-upgrade.txt" + ) + + with t.observe("MIG-03"): + result = t.upgrade(initial) + t.eq(t.current_version(), initial, "回退后版本") + t.evidence(f"{target} -> {initial} images={result.images} changed={result.changed}", "MIG-rollback.txt") diff --git a/test/kindtest/tests/10_ray.py b/test/kindtest/tests/10_ray.py new file mode 100644 index 0000000..8859596 --- /dev/null +++ b/test/kindtest/tests/10_ray.py @@ -0,0 +1,143 @@ +"""REG: cross-cluster Ray - one 测试流程 with four 测试观测点. + +The flow (procedure) runs *once*: it creates the Job, waits it Ready and tears it +down. Each ``with t.observe(...)`` is an observation point that produces one +report row. The Job uses the redesigned contract +(placement/runtime/workload/carrier), matching the deployed target schema. +""" + +from __future__ import annotations + +from kindtest import group + +reg = group("REG", title="基本功能回归") + +RAY_JOB = """ +apiVersion: rlinf.io/v1alpha1 +kind: Job +metadata: + name: {{runId}}-ray + labels: + {{labelKey}}: {{runId}}-ray +spec: + domain: {{domain}} + tasks: + - name: head + head: true + role: Actor + placement: + nodeSelector: + kubernetes.io/hostname: {{nodeA}} + runtime: + kind: Kubernetes + workload: + kind: rlinf.io/ray + version: v1 + parameters: + role: head + carrier: + type: rlinf.io/kubernetes-statefulset + version: v1 + parameters: + replicas: 1 + template: + spec: + containers: + - name: main + image: {{taskImage}} + imagePullPolicy: IfNotPresent + - name: worker1 + role: Env + placement: + nodeSelector: + kubernetes.io/hostname: {{nodeB}} + runtime: + kind: Kubernetes + workload: + kind: rlinf.io/ray + version: v1 + parameters: + role: worker + carrier: + type: rlinf.io/kubernetes-statefulset + version: v1 + parameters: + replicas: 1 + template: + spec: + containers: + - name: main + image: {{taskImage}} + imagePullPolicy: IfNotPresent + - name: worker2 + role: Env + placement: + nodeSelector: + kubernetes.io/hostname: {{nodeB2}} + runtime: + kind: Kubernetes + workload: + kind: rlinf.io/ray + version: v1 + parameters: + role: worker + carrier: + type: rlinf.io/kubernetes-statefulset + version: v1 + parameters: + replicas: 1 + template: + spec: + containers: + - name: main + image: {{taskImage}} + imagePullPolicy: IfNotPresent +""" + + +@reg.flow( + "跨集群 StatefulSet Ray", + items=[ + ("REG-01", "Ray Active", "head+2 worker，3 Active"), + ("REG-02", "rank 和放置", "head 与两 worker 落在 config.placement 指定的节点"), + ("REG-03", "Domain DNS", "worker 能解析 head 的 .rlark-domain 名称"), + ("REG-04", "跨集群连接", "两 worker 到 head 6379 成功"), + ], +) +def cross_cluster_ray(t): + job = f"{t.run_id}-ray" + head = t.head_pod(job) + workers = [f"{job}-worker1-0", f"{job}-worker2-0"] + + # -- 流程步骤：只建立/销毁一次集群，观测点复用 -- + t.delete_job(job) + t.apply(t.render(RAY_JOB), on="mgmt") + try: + t.require(lambda: t.kexists("pod", head, on="data", cluster="a"), + timeout=300, desc=f"{head} 出现") + t.wait_pod("a", head, 300) + for pod in workers: + t.require(lambda p=pod: t.kexists("pod", p, on="data", cluster="b"), + timeout=300, desc=f"{pod} 出现") + t.wait_pod("b", pod, 300) + t.sleep(20) # let Ray form the cluster + + with t.observe("REG-01"): + t.evidence(t.kexec("a", head, "ray", "status").combined(), "REG-ray-status.txt") + t.ge(t.ray_active("a", head), 3, "Ray Active") + + with t.observe("REG-02"): + t.eq(t.pod_node("a", head), t.config.get("placement.a"), "head 节点") + t.eq([t.pod_node("b", p) for p in workers], + [t.config.get("placement.b"), t.config.get("placement.b2")], "worker 节点") + + with t.observe("REG-03"): + resolved = t.dns_resolve("b", workers[0], f"{head}.rlark-domain") + t.evidence(f"{head}.rlark-domain -> {resolved}", "REG-dns.txt") + t.nonempty(resolved, "解析结果") + + with t.observe("REG-04"): + t.eq([t.tcp_connect("b", p, f"{head}.rlark-domain", 6379) for p in workers], + [True, True], "worker→head 连接") + finally: + t.delete_job(job) diff --git a/test/kindtest/tests/20_task.py b/test/kindtest/tests/20_task.py new file mode 100644 index 0000000..c7a6e0d --- /dev/null +++ b/test/kindtest/tests/20_task.py @@ -0,0 +1,103 @@ +"""TASK: single-task lifecycle as one 测试流程 with three sequential 测试观测点. + +Shows a procedure that mutates the environment between observation points +(create -> stop/start -> delete) while keeping one Job for the whole flow. +""" + +from __future__ import annotations + +from kindtest import group + +task = group("TASK", title="生命周期和错误恢复") + +SINGLE_TASK_JOB = """ +apiVersion: rlinf.io/v1alpha1 +kind: Job +metadata: + name: {{runId}}-single + labels: + {{labelKey}}: {{runId}}-single +spec: + domain: {{domain}} + tasks: + - name: main + head: true + role: Actor + placement: + nodeSelector: + kubernetes.io/hostname: {{nodeA}} + runtime: + kind: Kubernetes + workload: + kind: rlinf.io/ray + version: v1 + parameters: + role: head + carrier: + type: rlinf.io/kubernetes-statefulset + version: v1 + parameters: + replicas: 1 + template: + spec: + containers: + - name: main + image: {{taskImage}} + imagePullPolicy: IfNotPresent +""" + + +@task.flow( + "单任务生命周期", + items=[ + ("TASK-01", "新建并 Running", "Task phase=Running，Pod Ready"), + ("TASK-02", "stop/start", "stop 删除子 Task/Pod，start 按同名 Job 重建并 Running"), + ("TASK-03", "删除完整性", "删除 Job 后 Task 与 Pod 均回收"), + ], +) +def single_task_lifecycle(t): + job = f"{t.run_id}-single" + task_name = t.task_name(job) + pod = f"{job}-main-0" + ns = t.mgmt_ns("a") + + t.delete_job(job) + t.apply(t.render(SINGLE_TASK_JOB), on="mgmt") + try: + t.require(lambda: t.kexists("task", task_name, on="mgmt", ns=ns), + timeout=180, desc="Task 出现") + t.require(lambda: t.kexists("pod", pod, on="data", cluster="a"), + timeout=300, desc="Pod 出现") + t.wait_pod("a", pod, 300) + + with t.observe("TASK-01"): + t.eq(t.task_phase("a", task_name), "Running", "Task phase") + t.eq(t.pod_ready("a", pod), True, "Pod Ready") + + with t.observe("TASK-02"): + uid_before = t.task_uid("a", task_name) + t.job_stopped(job, True) + t.require(lambda: not t.kexists("pod", pod, on="data", cluster="a"), + timeout=200, desc="Pod 消失") + # 文档契约（concepts.md「Job 停止/启动」）：停止会删除子 Task CR， + # 启动时按同名 Job 重建新的 Task 与 Worker，因此 Task UID 会变化。 + t.require(lambda: not t.kexists("task", task_name, on="mgmt", ns=ns), + timeout=200, desc="Task 回收") + t.job_stopped(job, False) + t.require(lambda: t.kexists("task", task_name, on="mgmt", ns=ns), + timeout=300, desc="Task 重建") + t.require(lambda: t.kexists("pod", pod, on="data", cluster="a"), + timeout=300, desc="Pod 恢复") + t.wait_pod("a", pod, 300) + t.truthy(t.task_uid("a", task_name) != uid_before, + "Task 按文档重建（UID 变化）") + + with t.observe("TASK-03"): + t.delete_job(job) + t.require(lambda: not t.kexists("task", task_name, on="mgmt", ns=ns), + timeout=200, desc="Task 回收") + t.require(lambda: not t.kexists("pod", pod, on="data", cluster="a"), + timeout=200, desc="Pod 回收") + t.truthy(True, "Job/Task/Pod 均已回收") + finally: + t.delete_job(job) diff --git a/test/kindtest/tests/90_cleanup.py b/test/kindtest/tests/90_cleanup.py new file mode 100644 index 0000000..b4f764c --- /dev/null +++ b/test/kindtest/tests/90_cleanup.py @@ -0,0 +1,63 @@ +"""CLEAN: remove run-owned objects and prove residual=0 without touching others. + +Cleanup is written as ordinary cases so it is reported in the same matrix and so +``--only CLEAN`` can be run on its own after an interrupted run. +""" + +from __future__ import annotations + +from kindtest import group + +clean = group("CLEAN", title="清理和最终状态") + + +def _owned(names: list[str], prefix: str) -> list[str]: + return sorted(n for n in names if n.startswith(prefix)) + + +@clean.case("CLEAN-01", "测试 Job 清理", expect="本轮前缀 Job 全部删除，残留为 0") +def clean_jobs(t): + owned = _owned(t.mgmt.names("jobs"), t.run_id) + for name in owned: + t.mgmt.delete("job", name, wait=False) + if owned: + t.sleep(10) + left = _owned(t.mgmt.names("jobs"), t.run_id) + return t.eq(left, [], f"残留 Job（本轮共删除 {len(owned)}）") + + +@clean.case("CLEAN-02", "测试 Task 和 Workload 清理", expect="两数据面均无本轮对象") +def clean_workloads(t): + for key in t.data_keys(): + cl = t.cl(key) + tasks = _owned(cl.mgmt.names("tasks"), t.run_id) + workloads = [] + for kind in ("statefulset", "deployment", "daemonset", "pod"): + workloads += _owned(cl.data.names(kind), t.run_id) + assert not tasks, f"{cl.name} 残留 Task: {tasks}" + assert not workloads, f"{cl.name} 残留 Workload: {workloads}" + return "两数据面 Task/StatefulSet/Deployment/Pod 残留为 0" + + +@clean.case("CLEAN-03", "无关既有任务", expect="非本轮对象不受清理影响") +def unrelated_preserved(t): + others = [n for n in t.mgmt.names("jobs") if not n.startswith(t.run_id)] + t.evidence("非本轮 Job: " + (", ".join(others) or "(无)"), "CLEAN-unrelated-jobs.txt") + return t.truthy(True, f"非本轮 Job 保留 {len(others)} 个") + + +@clean.case("CLEAN-04", "最终 Agent 镜像", expect="两侧 Cluster Agent 报告目标镜像且 Ready") +def final_agents(t): + facts = [] + for key in t.data_keys(): + cl = t.cl(key) + image = cl.data.jp("deploy", "rlark-agent", "{.spec.template.spec.containers[0].image}") + ready = cl.data.jp("deploy", "rlark-agent", "{.status.readyReplicas}/{.spec.replicas}") + _all_ready(ready, f"{cl.name} Cluster Agent") + facts.append(f"{cl.name}={image}") + return "；".join(facts) + + +def _all_ready(ratio: str, what: str) -> None: + got, _, want = ratio.partition("/") + assert got and want and got == want, f"{what} 未就绪: {ratio or 'n/a'}"