From e8ba4f5c8019d7899a0ce025fc6afab767d6f187 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:04:52 -0700 Subject: [PATCH 01/35] add base image var --- Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Makefile b/Makefile index d423967f4..34d186890 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,12 @@ TAG ?= $(GIT_TAG) ARCH ?= linux/amd64 MANAGER_IMG ?= $(REGISTRY)/ome-manager:$(TAG) +# Optional final-stage base for manager/model-agent/ome-agent Dockerfiles (default oraclelinux:10-slim). +# Apple Silicon + --platform=linux/amd64 uses QEMU; OL10 glibc can fail with "CPU does not support x86-64-v3". +# For local amd64 builds on Mac, use: BASE_IMAGE=ubuntu:24.04 make ome-image +BASE_IMAGE ?= +BASE_IMAGE_DOCKER_ARGS = $(if $(BASE_IMAGE),--build-arg BASE_IMAGE=$(BASE_IMAGE),) + # Git version and commit information for build version_pkg = github.com/sgl-project/ome/pkg/version GIT_TAG ?= $(shell git describe --tags --dirty --always) @@ -332,6 +338,7 @@ run-ome-agent-replica: fmt vet ome-agent ## Run ome-agent binary from local host ome-image: fmt vet ## Build ome-manager image. @echo "🚀 Building ome-manager image..." $(DOCKER_BUILD_CMD) build --platform=$(ARCH) \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -342,6 +349,7 @@ ome-image: fmt vet ## Build ome-manager image. model-agent-image: fmt vet ## Build model-agent image. @echo "🚀 Building model-agent image..." $(DOCKER_BUILD_CMD) build --platform=$(ARCH) \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -362,6 +370,7 @@ multinode-prober-image: fmt vet ## Build multinode-prober image. ome-agent-image: fmt vet xet-build ## Build ome-agent image. @echo "🚀 Building ome-agent image..." $(DOCKER_BUILD_CMD) build --platform=$(ARCH) \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -388,11 +397,13 @@ build-all-images: fmt vet ## 🚀 Build all images for current architecture build-all-images-multiarch: fmt vet docker-buildx-setup ## 🌍 Build all images for multiple architectures @echo "🌍 Building all OME images for linux/amd64,linux/arm64..." $(DOCKER_BUILD_CMD) buildx build --platform=linux/amd64,linux/arm64 \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ . -f dockerfiles/manager.Dockerfile -t $(MANAGER_IMG) --push $(DOCKER_BUILD_CMD) buildx build --platform=linux/amd64,linux/arm64 \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -403,6 +414,7 @@ build-all-images-multiarch: fmt vet docker-buildx-setup ## 🌍 Build all images --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ . -f dockerfiles/multinode-prober.Dockerfile -t $(REGISTRY)/multinode-prober:$(TAG) --push $(DOCKER_BUILD_CMD) buildx build --platform=linux/amd64,linux/arm64 \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ From a99418cb82de9d532f3e88480ea5374c4a6019ea Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:12:13 -0700 Subject: [PATCH 02/35] add note about ome.io/skip-model-ready-node-selector --- charts/ome-resources/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/charts/ome-resources/README.md b/charts/ome-resources/README.md index 60989a56d..59959b78b 100644 --- a/charts/ome-resources/README.md +++ b/charts/ome-resources/README.md @@ -90,3 +90,21 @@ OME Resources and Controller | ome.omeAgent.tag | string | `"v0.1.2"` | | | ome.omeAgent.vaultId | string | `"ocid1.vault.oc1.ap-osaka-1.dummy.dummy-vault"` | | | ome.version | string | `"v0.1.2"` | | + +## Autoscaling (Karpenter / cluster autoscaler) + +Engine and decoder pods normally get a **required** `nodeSelector` of the form `models.ome.io/clusterbasemodel.=Ready` (or the namespace-scoped base-model equivalent). The model-agent DaemonSet applies that label only **after** weights are on disk. On elastic GPU pools, autoscalers such as Karpenter may refuse to provision a node when the pod requires a label that no template advertises yet, which can deadlock cold starts. + +**Opt-in:** set this annotation on the `InferenceService` to **omit** the model-ready `nodeSelector` (accelerator / runtime merged selectors still apply): + +```yaml +metadata: + annotations: + ome.io/skip-model-ready-node-selector: "true" +``` + +**Semantics:** pods may schedule on a GPU node before the model is present on the host; you rely on hostPath + model-agent download, container restarts, or similar until the model is available. GPU pool labels, taints, and tolerations must still align (for example `gpu=true` and `nvidia.com/gpu` tolerations). + +**BenchmarkJob:** if the job references an `InferenceService`, the same annotation on that `InferenceService` controls whether the benchmark pod gets the model-ready selector. + +Multi-node or PD-disaggregated behavior is unchanged; this only affects the optional model-ready scheduling constraint for cluster- or namespace-scoped base models. From 2c2df8ae69bd9c819ace1a0373a740a9b438f3db Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:12:40 -0700 Subject: [PATCH 03/35] add SkipModelReadyNodeSelectorAnnotationKey --- pkg/constants/constants.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 171fc8582..498c4c89d 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -335,6 +335,11 @@ const ( InferenceServiceLabel = "ome.io/inferenceservice" ) +// SkipModelReadyNodeSelectorAnnotationKey, when set to "true" on an InferenceService, omits the +// models.ome.io/...=Ready nodeSelector from engine and decoder pods. Use this with elastic GPU +// pools (e.g. Karpenter) where nodes are provisioned before the model-agent applies the Ready label. +const SkipModelReadyNodeSelectorAnnotationKey = "ome.io/skip-model-ready-node-selector" + // InferenceService default/canary constants const ( InferenceServiceDefault = "default" From 179a5b98e803cff0490b284675a9cc04eb905cca Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:13:32 -0700 Subject: [PATCH 04/35] add conditional for skipping model label when ome.io/skip-model-ready-node-selector set --- .../v1beta1/benchmark/controller.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/controller/v1beta1/benchmark/controller.go b/pkg/controller/v1beta1/benchmark/controller.go index 705a1c60f..5fa0117c0 100644 --- a/pkg/controller/v1beta1/benchmark/controller.go +++ b/pkg/controller/v1beta1/benchmark/controller.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" + "github.com/sgl-project/ome/pkg/constants" "github.com/sgl-project/ome/pkg/controller/v1beta1/benchmark/reconcilers/job" benchmarkutils "github.com/sgl-project/ome/pkg/controller/v1beta1/benchmark/utils" "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" @@ -247,11 +248,19 @@ func (r *BenchmarkJobReconciler) addNodeSelectorFromInferenceService(ctx context return err } - isvcutils.AddNodeSelectorForModelReadyNode(podSpec, baseModelMeta) - r.Log.Info("Added node selector for benchmark job", - "baseModel", baseModelMeta.Name, - "namespace", baseModelMeta.Namespace, - "benchmarkJob", benchmarkJob.Name) + if isvcutils.IsSkipModelReadyNodeSelector(inferenceService.Annotations) { + r.Log.Info("Skipping model-ready nodeSelector for benchmark job (InferenceService annotation)", + "annotation", constants.SkipModelReadyNodeSelectorAnnotationKey, + "inferenceService", inferenceService.Namespace+"/"+inferenceService.Name, + "benchmarkJob", benchmarkJob.Name, + "baseModel", baseModelMeta.Name) + } else { + isvcutils.AddNodeSelectorForModelReadyNode(podSpec, baseModelMeta) + r.Log.Info("Added model-ready nodeSelector for benchmark job", + "baseModel", baseModelMeta.Name, + "namespace", baseModelMeta.Namespace, + "benchmarkJob", benchmarkJob.Name) + } return nil } From 638d74530cb2e52290dda807c08a41880fe75fe9 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:14:31 -0700 Subject: [PATCH 05/35] update log to print nodeSelector message --- .../v1beta1/inferenceservice/components/base.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/components/base.go b/pkg/controller/v1beta1/inferenceservice/components/base.go index efe00c40a..a457ba96a 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base.go @@ -204,8 +204,15 @@ func UpdatePodSpecNodeSelector(b *BaseComponentFields, isvc *v1beta1.InferenceSe return } - // Add preferred node affinity for model readiness using the shared utility function - isvcutils.AddNodeSelectorForModelReadyNode(podSpec, b.BaseModelMeta) + // Optional: omit models.ome.io/...=Ready so autoscalers can provision GPU nodes before the + // model-agent labels them (see SkipModelReadyNodeSelectorAnnotationKey). + if isvcutils.IsSkipModelReadyNodeSelector(isvc.Annotations) { + b.Log.Info("Skipping model-ready nodeSelector per InferenceService annotation", + "annotation", constants.SkipModelReadyNodeSelectorAnnotationKey, + "inferenceService", isvc.Name, "namespace", isvc.Namespace) + } else { + isvcutils.AddNodeSelectorForModelReadyNode(podSpec, b.BaseModelMeta) + } // Add node selector merged from AcceleratorClass if applicable // Only add mergedNodeSelector to engine and decoder component. @@ -219,7 +226,7 @@ func UpdatePodSpecNodeSelector(b *BaseComponentFields, isvc *v1beta1.InferenceSe } } - b.Log.Info("Added preferred node affinity for model scheduling", + b.Log.Info("Updated pod nodeSelector for model scheduling", "modelName", b.BaseModelMeta.Name, "namespace", b.BaseModelMeta.Namespace, "inferenceService", isvc.Name) From ddc869a486378be7fa2199ebf338a61a28af952b Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:15:23 -0700 Subject: [PATCH 06/35] impliment TestIsSkipModelReadyNodeSelector --- pkg/controller/v1beta1/inferenceservice/utils/utils.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils.go b/pkg/controller/v1beta1/inferenceservice/utils/utils.go index 31b877e36..f2c2d2ef2 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils.go @@ -94,6 +94,16 @@ func GetTargetServicePort(ctx context.Context, c client.Client, isvc *v1beta1.In return port, nil } +// IsSkipModelReadyNodeSelector reports whether the InferenceService annotation requests omitting +// the models.ome.io/...=Ready nodeSelector (for autoscaling / cold-start scheduling). +func IsSkipModelReadyNodeSelector(annotations map[string]string) bool { + if annotations == nil { + return false + } + v, ok := annotations[constants.SkipModelReadyNodeSelectorAnnotationKey] + return ok && v == "true" +} + // AddNodeSelectorForModelReadyNode adds a node selector to the pod spec // for scheduling pods on nodes where the base model is ready. // This is used by both InferenceService and BenchmarkJob controllers to ensure pods From 2ffcce76c88ce4f5435725e4cb0eba57c9bdf996 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:16:24 -0700 Subject: [PATCH 07/35] add test for SkipModelReadyNodeSelectorAnnotationKey --- .../inferenceservice/components/base_test.go | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/components/base_test.go b/pkg/controller/v1beta1/inferenceservice/components/base_test.go index 9de3eae38..289b80061 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base_test.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base_test.go @@ -22,6 +22,7 @@ func TestUpdatePodSpecNodeSelector(t *testing.T) { baseModelMeta *metav1.ObjectMeta fineTunedServingWithMergedWeights bool existingNodeSelector map[string]string + isvcAnnotations map[string]string expectedLabelKey string expectNodeSelector bool }{ @@ -90,6 +91,39 @@ func TestUpdatePodSpecNodeSelector(t *testing.T) { baseModelMeta: nil, expectNodeSelector: false, }, + { + name: "Skip model-ready nodeSelector when InferenceService annotation is true", + baseModel: &v1beta1.BaseModelSpec{ + ModelFormat: v1beta1.ModelFormat{ + Name: "safetensors", + }, + }, + baseModelMeta: &metav1.ObjectMeta{ + Name: "my-model", + Namespace: "", + }, + isvcAnnotations: map[string]string{ + constants.SkipModelReadyNodeSelectorAnnotationKey: "true", + }, + expectNodeSelector: false, + }, + { + name: "Annotation false still adds model-ready nodeSelector", + baseModel: &v1beta1.BaseModelSpec{ + ModelFormat: v1beta1.ModelFormat{ + Name: "safetensors", + }, + }, + baseModelMeta: &metav1.ObjectMeta{ + Name: "my-model", + Namespace: "", + }, + isvcAnnotations: map[string]string{ + constants.SkipModelReadyNodeSelectorAnnotationKey: "false", + }, + expectedLabelKey: "models.ome.io/clusterbasemodel.my-model", + expectNodeSelector: true, + }, { name: "Long model names should be handled", baseModel: &v1beta1.BaseModelSpec{ @@ -128,13 +162,14 @@ func TestUpdatePodSpecNodeSelector(t *testing.T) { // Create inference service isvc := &v1beta1.InferenceService{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-isvc", - Namespace: "default", + Name: "test-isvc", + Namespace: "default", + Annotations: tt.isvcAnnotations, }, } // Call the function - UpdatePodSpecNodeSelector(b, isvc, podSpec, "") + UpdatePodSpecNodeSelector(b, isvc, podSpec, v1beta1.EngineComponent) // Verify the result if !tt.expectNodeSelector { From 633389eb62222cf49efe1d3fa8d4f43b1a927faa Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 31 Mar 2026 17:16:46 -0700 Subject: [PATCH 08/35] impliment TestIsSkipModelReadyNodeSelector --- .../inferenceservice/utils/utils_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go b/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go index 4fdd72246..15b5485a3 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go @@ -1870,6 +1870,25 @@ func TestGetTargetServicePort_ServiceNameResolution(t *testing.T) { } } +func TestIsSkipModelReadyNodeSelector(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + want bool + }{ + {name: "nil annotations", annotations: nil, want: false}, + {name: "empty annotations", annotations: map[string]string{}, want: false}, + {name: "true", annotations: map[string]string{constants.SkipModelReadyNodeSelectorAnnotationKey: "true"}, want: true}, + {name: "false is not skip", annotations: map[string]string{constants.SkipModelReadyNodeSelectorAnnotationKey: "false"}, want: false}, + {name: "other value is not skip", annotations: map[string]string{constants.SkipModelReadyNodeSelectorAnnotationKey: "yes"}, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsSkipModelReadyNodeSelector(tt.annotations)) + }) + } +} + func TestAddNodeSelectorForReadyModel(t *testing.T) { tests := []struct { name string From dbbcd692e370d3a78865d5b6e359bdc68f2aba6a Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Wed, 1 Apr 2026 09:16:32 -0700 Subject: [PATCH 09/35] ci(vsco): add fork-only workflow to publish ome-manager to ghcr.io/vsco Self-contained workflow (workflow_dispatch). Delete this file before PR to sgl-project/ome. Does not modify upstream dev-images/release workflows. Made-with: Cursor --- .../workflows/vsco-publish-ome-manager.yaml | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .github/workflows/vsco-publish-ome-manager.yaml diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml new file mode 100644 index 000000000..350a1e300 --- /dev/null +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -0,0 +1,96 @@ +# ============================================================================= +# VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO sgl-project/ome +# +# This workflow builds dockerfiles/manager.Dockerfile and pushes to: +# ghcr.io/vsco/ome-manager: +# +# Delete this entire file when upstreaming your fork (or keep it only on a +# long-lived internal branch). Upstream does not need VSCO registry automation. +# +# Usage: +# Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow +# Set "image_tag" (e.g. v0.1.4-vsco2) or leave default to use short SHA. +# +# Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. +# If pushes fail with 403, add a repo secret VSCO_GHCR_TOKEN (PAT with +# write:packages) and uncomment the password line below. +# ============================================================================= + +name: VSCO (fork) publish ome-manager to ghcr.io/vsco + +on: + workflow_dispatch: + inputs: + image_tag: + description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' + required: false + default: '' + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_ORG: vsco + IMAGE_NAME: ome-manager + GO_VERSION: '1.25' + +jobs: + build-push-ome-manager: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + # Optional: set repo secret VSCO_GHCR_TOKEN (PAT, write:packages) if GITHUB_TOKEN cannot push + password: ${{ secrets.VSCO_GHCR_TOKEN || secrets.GITHUB_TOKEN }} + + - name: Compute image tag + id: tag + run: | + if [ -n "${{ inputs.image_tag }}" ]; then + TAG="${{ inputs.image_tag }}" + else + TAG="sha-$(echo '${{ github.sha }}' | cut -c1-7)" + fi + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "Publishing ${REGISTRY}/${IMAGE_ORG}/${IMAGE_NAME}:${TAG}" + + - name: Build and push ome-manager (linux/amd64) + uses: docker/build-push-action@v7 + with: + context: . + file: dockerfiles/manager.Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + VERSION=${{ steps.tag.outputs.tag }} + GIT_TAG=${{ steps.tag.outputs.tag }} + GIT_COMMIT=${{ github.sha }} + + - name: Summary + run: | + echo "## VSCO ome-manager image" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Pushed: \`${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Remove \`.github/workflows/vsco-publish-ome-manager.yaml\` before submitting a PR to sgl-project/ome." >> "$GITHUB_STEP_SUMMARY" From 37a27d847cb543730c6bb1f148200708ff64a491 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Wed, 1 Apr 2026 10:05:17 -0700 Subject: [PATCH 10/35] update to use vsco ghcr for now --- .github/workflows/vsco-publish-ome-manager.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 350a1e300..576143332 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -1,15 +1,17 @@ # ============================================================================= # VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO sgl-project/ome # -# This workflow builds dockerfiles/manager.Dockerfile and pushes to: +# This workflow builds dockerfiles/manager.Dockerfile and pushes ONLY to: # ghcr.io/vsco/ome-manager: +# (Not ghcr.io/moirai-internal — that is upstream Makefile / other workflows.) # # Delete this entire file when upstreaming your fork (or keep it only on a # long-lived internal branch). Upstream does not need VSCO registry automation. # # Usage: # Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -# Set "image_tag" (e.g. v0.1.4-vsco2) or leave default to use short SHA. +# Default tag matches local manual pushes: v0.1.4-vsco1. Override "image_tag" +# for a new release, or clear it to use git short SHA instead. # # Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. # If pushes fail with 403, add a repo secret VSCO_GHCR_TOKEN (PAT with @@ -24,7 +26,7 @@ on: image_tag: description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' required: false - default: '' + default: 'v0.1.4-vsco1' permissions: contents: read @@ -52,7 +54,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Log in to GitHub Container Registry + - name: Log in to ghcr.io (org vsco packages) uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} @@ -63,13 +65,12 @@ jobs: - name: Compute image tag id: tag run: | - if [ -n "${{ inputs.image_tag }}" ]; then - TAG="${{ inputs.image_tag }}" - else + TAG="${{ inputs.image_tag }}" + if [ -z "$TAG" ]; then TAG="sha-$(echo '${{ github.sha }}' | cut -c1-7)" fi echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - echo "Publishing ${REGISTRY}/${IMAGE_ORG}/${IMAGE_NAME}:${TAG}" + echo "Publishing to ghcr.io/vsco → ${REGISTRY}/${IMAGE_ORG}/${IMAGE_NAME}:${TAG}" - name: Build and push ome-manager (linux/amd64) uses: docker/build-push-action@v7 From 3ebc0a37668e1f2b1ec738e9b1cfd9c1659eb330 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Wed, 1 Apr 2026 10:20:05 -0700 Subject: [PATCH 11/35] update tag to v0.1.4-vsco2 --- .github/workflows/vsco-publish-ome-manager.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 576143332..35df71483 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ # # Usage: # Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -# Default tag matches local manual pushes: v0.1.4-vsco1. Override "image_tag" +# Default tag matches local manual pushes: v0.1.4-vsco2. Override "image_tag" # for a new release, or clear it to use git short SHA instead. # # Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ on: image_tag: description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' required: false - default: 'v0.1.4-vsco1' + default: 'v0.1.4-vsco2' permissions: contents: read From 4eb1629c6e944c8101ced4d8490fbb3dd8a59921 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Thu, 2 Apr 2026 11:22:44 -0700 Subject: [PATCH 12/35] update to use vsco standardized versioning scheme --- charts/ome-crd/Chart.yaml | 4 ++-- charts/ome-resources/Chart.yaml | 5 +++-- charts/ome-serving/Chart.yaml | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/charts/ome-crd/Chart.yaml b/charts/ome-crd/Chart.yaml index 1707234ae..3df4bdfca 100644 --- a/charts/ome-crd/Chart.yaml +++ b/charts/ome-crd/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ome-crd description: OME Custom Resource Definitions type: application -version: 0.1.0 -appVersion: "1.16.0" +version: 0.1.4 +appVersion: "0.1.4-vsco1" diff --git a/charts/ome-resources/Chart.yaml b/charts/ome-resources/Chart.yaml index e7c14d1e3..9be6cdcf6 100644 --- a/charts/ome-resources/Chart.yaml +++ b/charts/ome-resources/Chart.yaml @@ -2,5 +2,6 @@ apiVersion: v2 name: ome-resources description: OME Resources and Controller type: application -version: 0.1.0 -appVersion: "1.16.0" +# Helm chart version: strict semver (no leading v). Keep in sync with release / VSCO publish workflows. +version: 0.1.4 +appVersion: "v0.1.4-vsco1" diff --git a/charts/ome-serving/Chart.yaml b/charts/ome-serving/Chart.yaml index 81e148109..27e8803ed 100644 --- a/charts/ome-serving/Chart.yaml +++ b/charts/ome-serving/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: ome-serving description: OME Model Serving - ClusterBaseModels, ClusterServingRuntimes, and InferenceServices type: application -version: 0.1.0 -appVersion: "1.0.0" +version: 0.1.4 +appVersion: "v0.1.4-vsco1" keywords: - ome - inference From 512c9638e1ef073ba44b19aaae72697a98420a86 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Thu, 2 Apr 2026 11:23:19 -0700 Subject: [PATCH 13/35] update to use vsco standardized versioning scheme; define instanceTypeMap --- charts/ome-resources/values.yaml | 63 ++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index cb4f96397..50d522dd3 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -1,14 +1,12 @@ -# Global settings that apply to all resources +# VSCO fork: Helm charts are published to oci://ghcr.io/vsco/charts (see .github/workflows/vsco-publish-helm-charts.yaml). +# shared-infra only sets var.ome_version (chart semver). Forked ome-manager uses a full image ref so other images can stay on global.hub. global: - # Image pull secrets for all containers in the chart - # Example: - # imagePullSecrets: - # - name: my-registry-secret hub: "ghcr.io/moirai-internal" - imagePullSecrets: [] + imagePullSecrets: + - name: ghcr-creds ome: - version: &defaultVersion v0.1.4 + version: &defaultVersion v0.1.4-vsco1 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" @@ -55,7 +53,8 @@ ome: tolerations: [] topologySpreadConstraints: [] affinity: {} - image: ome-manager + # Full path → ome.imageWithHub ignores global.hub (VSCO-built controller) + image: ghcr.io/vsco/ome-manager tag: *defaultVersion resources: limits: @@ -102,6 +101,54 @@ modelAgent: pullPolicy: Always tag: *defaultVersion + # Instance type → GPU label map for model-agent (JSON in ConfigMap key instance-type-map). + # Extend or replace per cluster; aligns with shared-infra AWS GPU pools when published from this fork. + instanceTypeMap: + "BM.GPU.A10.4": "A10" + "BM.GPU.A100-v2.8": "A100-80G" + "BM.GPU4.8": "A100-40G" + "BM.GPU.B4.8": "A100-40G" + "BM.GPU.H100.8": "H100" + "BM.GPU.H100-NC.8": "H100" + "BM.GPU.H200.8": "H200" + "BM.GPU.H200-NC.8": "H200" + "BM.GPU.B200.8": "B200" + "p5.48xlarge": "H100" + "Standard_ND96isr_H100_v5": "H100" + "a3-highgpu-8g": "H100" + "gd-8xh100ib-i128": "H100" + "gd-8xh200ib-i128": "H200" + "gd-8xl40-i128": "L40" + "gpu-h100-sxm": "H100" + "gpu-h200-sxm": "H200" + "gpu-b200-sxm": "B200" + "gpu-l40s": "L40S" + "g4dn.xlarge": "T4" + "g4dn.2xlarge": "T4" + "g4dn.4xlarge": "T4" + "g4dn.8xlarge": "T4" + "g4dn.12xlarge": "T4" + "g4dn.16xlarge": "T4" + "g4dn.metal": "T4" + "g6e.xlarge": "L40S" + "g6e.2xlarge": "L40S" + "g6e.4xlarge": "L40S" + "g6e.8xlarge": "L40S" + "g6e.12xlarge": "L40S" + "g6e.16xlarge": "L40S" + "g6e.24xlarge": "L40S" + "g6e.48xlarge": "L40S" + "g5.xlarge": "A10G" + "g5.2xlarge": "A10G" + "g5.4xlarge": "A10G" + "g5.8xlarge": "A10G" + "g5.12xlarge": "A10G" + "g5.16xlarge": "A10G" + "g5.24xlarge": "A10G" + "g5.48xlarge": "A10G" + "p4d.24xlarge": "A100-40G" + "p4de.24xlarge": "A100-80G" + # When enabled, the model agent will only run on nodes with GPU gpuNodesOnly: false From bf8701a9b4f06300441324d4a594bacde83a27e1 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Thu, 2 Apr 2026 11:24:15 -0700 Subject: [PATCH 14/35] migrate instanceTypeMap to generate from charts/ome-resources/values.yaml --- .../model-agent-daemonset/configmap.yaml | 31 ++----------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml b/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml index 2d0026701..47ce0e4d5 100644 --- a/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml +++ b/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml @@ -4,32 +4,7 @@ metadata: name: model-agent-config-map namespace: {{ .Release.Namespace }} data: - # Instance type mappings for various cloud providers: - # - Oracle Cloud (OCI) shapes - # - AWS instance types - # - Azure instance types - # - Google Cloud instance types - # - CoreWeave instance types - # - Nebius instance types + # Instance type → GPU name mapping (AWS, OCI, Azure, GCP, CoreWeave, Nebius, …). + # Configure via .Values.modelAgent.instanceTypeMap (map of string → string, JSON-serialized). instance-type-map: |- - { - "BM.GPU.A10.4": "A10", - "BM.GPU.A100-v2.8": "A100-80G", - "BM.GPU4.8": "A100-40G", - "BM.GPU.B4.8": "A100-40G", - "BM.GPU.H100.8": "H100", - "BM.GPU.H100-NC.8": "H100", - "BM.GPU.H200.8": "H200", - "BM.GPU.H200-NC.8": "H200", - "BM.GPU.B200.8": "B200", - "p5.48xlarge": "H100", - "Standard_ND96isr_H100_v5": "H100", - "a3-highgpu-8g": "H100", - "gd-8xh100ib-i128": "H100", - "gd-8xh200ib-i128": "H200", - "gd-8xl40-i128": "L40", - "gpu-h100-sxm": "H100", - "gpu-h200-sxm": "H200", - "gpu-b200-sxm": "B200", - "gpu-l40s": "L40S" - } +{{- toJson .Values.modelAgent.instanceTypeMap | nindent 4 }} From 24ce090b7d014bf6966b367d502a9c0e3783352f Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Mon, 6 Apr 2026 10:46:32 -0700 Subject: [PATCH 15/35] add descriptino for case with adding annotation to skip label for autoscaling --- pkg/controller/v1beta1/benchmark/controller.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/controller/v1beta1/benchmark/controller.go b/pkg/controller/v1beta1/benchmark/controller.go index 5fa0117c0..6ccbb30ad 100644 --- a/pkg/controller/v1beta1/benchmark/controller.go +++ b/pkg/controller/v1beta1/benchmark/controller.go @@ -248,6 +248,11 @@ func (r *BenchmarkJobReconciler) addNodeSelectorFromInferenceService(ctx context return err } + // Model-ready node selector follows the InferenceService: when the skip annotation is absent or + // not "true", require models.ome.io/...=Ready so the benchmark runs on nodes where model-agent + // has already placed weights. When constants.SkipModelReadyNodeSelectorAnnotationKey is "true", + // omit that selector so the job can schedule like engine pods on elastic GPU nodes (e.g. + // Karpenter) before the Ready label exists. if isvcutils.IsSkipModelReadyNodeSelector(inferenceService.Annotations) { r.Log.Info("Skipping model-ready nodeSelector for benchmark job (InferenceService annotation)", "annotation", constants.SkipModelReadyNodeSelectorAnnotationKey, From 6cd67d469d4e5d364c38e3630a29b092a8ded360 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 2 Apr 2026 19:18:52 -0600 Subject: [PATCH 16/35] Revert the ome chart version updates as this is handled in the release. --- charts/ome-crd/Chart.yaml | 4 ++-- charts/ome-resources/Chart.yaml | 5 ++--- charts/ome-serving/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/charts/ome-crd/Chart.yaml b/charts/ome-crd/Chart.yaml index 3df4bdfca..1707234ae 100644 --- a/charts/ome-crd/Chart.yaml +++ b/charts/ome-crd/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: ome-crd description: OME Custom Resource Definitions type: application -version: 0.1.4 -appVersion: "0.1.4-vsco1" +version: 0.1.0 +appVersion: "1.16.0" diff --git a/charts/ome-resources/Chart.yaml b/charts/ome-resources/Chart.yaml index 9be6cdcf6..e7c14d1e3 100644 --- a/charts/ome-resources/Chart.yaml +++ b/charts/ome-resources/Chart.yaml @@ -2,6 +2,5 @@ apiVersion: v2 name: ome-resources description: OME Resources and Controller type: application -# Helm chart version: strict semver (no leading v). Keep in sync with release / VSCO publish workflows. -version: 0.1.4 -appVersion: "v0.1.4-vsco1" +version: 0.1.0 +appVersion: "1.16.0" diff --git a/charts/ome-serving/Chart.yaml b/charts/ome-serving/Chart.yaml index 27e8803ed..81e148109 100644 --- a/charts/ome-serving/Chart.yaml +++ b/charts/ome-serving/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: ome-serving description: OME Model Serving - ClusterBaseModels, ClusterServingRuntimes, and InferenceServices type: application -version: 0.1.4 -appVersion: "v0.1.4-vsco1" +version: 0.1.0 +appVersion: "1.0.0" keywords: - ome - inference From 0c71e71cefdf6812efe8caac54c89f40514d14e2 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 2 Apr 2026 19:19:58 -0600 Subject: [PATCH 17/35] Remove for the time being. --- .../workflows/vsco-publish-ome-manager.yaml | 186 +++++++++--------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 35df71483..7c0124ba2 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -1,97 +1,97 @@ -# ============================================================================= -# VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO sgl-project/ome +## ============================================================================= +## VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO sgl-project/ome +## +## This workflow builds dockerfiles/manager.Dockerfile and pushes ONLY to: +## ghcr.io/vsco/ome-manager: +## (Not ghcr.io/moirai-internal — that is upstream Makefile / other workflows.) +## +## Delete this entire file when upstreaming your fork (or keep it only on a +## long-lived internal branch). Upstream does not need VSCO registry automation. +## +## Usage: +## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow +## Default tag matches local manual pushes: v0.1.4-vsco2. Override "image_tag" +## for a new release, or clear it to use git short SHA instead. +## +## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. +## If pushes fail with 403, add a repo secret VSCO_GHCR_TOKEN (PAT with +## write:packages) and uncomment the password line below. +## ============================================================================= # -# This workflow builds dockerfiles/manager.Dockerfile and pushes ONLY to: -# ghcr.io/vsco/ome-manager: -# (Not ghcr.io/moirai-internal — that is upstream Makefile / other workflows.) +#name: VSCO (fork) publish ome-manager to ghcr.io/vsco # -# Delete this entire file when upstreaming your fork (or keep it only on a -# long-lived internal branch). Upstream does not need VSCO registry automation. +#on: +# workflow_dispatch: +# inputs: +# image_tag: +# description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' +# required: false +# default: 'v0.1.4-vsco2' # -# Usage: -# Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -# Default tag matches local manual pushes: v0.1.4-vsco2. Override "image_tag" -# for a new release, or clear it to use git short SHA instead. +#permissions: +# contents: read +# packages: write # -# Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. -# If pushes fail with 403, add a repo secret VSCO_GHCR_TOKEN (PAT with -# write:packages) and uncomment the password line below. -# ============================================================================= - -name: VSCO (fork) publish ome-manager to ghcr.io/vsco - -on: - workflow_dispatch: - inputs: - image_tag: - description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' - required: false - default: 'v0.1.4-vsco2' - -permissions: - contents: read - packages: write - -env: - REGISTRY: ghcr.io - IMAGE_ORG: vsco - IMAGE_NAME: ome-manager - GO_VERSION: '1.25' - -jobs: - build-push-ome-manager: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Setup Go - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - cache: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to ghcr.io (org vsco packages) - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - # Optional: set repo secret VSCO_GHCR_TOKEN (PAT, write:packages) if GITHUB_TOKEN cannot push - password: ${{ secrets.VSCO_GHCR_TOKEN || secrets.GITHUB_TOKEN }} - - - name: Compute image tag - id: tag - run: | - TAG="${{ inputs.image_tag }}" - if [ -z "$TAG" ]; then - TAG="sha-$(echo '${{ github.sha }}' | cut -c1-7)" - fi - echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - echo "Publishing to ghcr.io/vsco → ${REGISTRY}/${IMAGE_ORG}/${IMAGE_NAME}:${TAG}" - - - name: Build and push ome-manager (linux/amd64) - uses: docker/build-push-action@v7 - with: - context: . - file: dockerfiles/manager.Dockerfile - platforms: linux/amd64 - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - VERSION=${{ steps.tag.outputs.tag }} - GIT_TAG=${{ steps.tag.outputs.tag }} - GIT_COMMIT=${{ github.sha }} - - - name: Summary - run: | - echo "## VSCO ome-manager image" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Pushed: \`${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}\`" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Remove \`.github/workflows/vsco-publish-ome-manager.yaml\` before submitting a PR to sgl-project/ome." >> "$GITHUB_STEP_SUMMARY" +#env: +# REGISTRY: ghcr.io +# IMAGE_ORG: vsco +# IMAGE_NAME: ome-manager +# GO_VERSION: '1.25' +# +#jobs: +# build-push-ome-manager: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout +# uses: actions/checkout@v6 +# +# - name: Setup Go +# uses: actions/setup-go@v6 +# with: +# go-version: ${{ env.GO_VERSION }} +# cache: true +# +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v3 +# +# - name: Log in to ghcr.io (org vsco packages) +# uses: docker/login-action@v4 +# with: +# registry: ${{ env.REGISTRY }} +# username: ${{ github.actor }} +# # Optional: set repo secret VSCO_GHCR_TOKEN (PAT, write:packages) if GITHUB_TOKEN cannot push +# password: ${{ secrets.VSCO_GHCR_TOKEN || secrets.GITHUB_TOKEN }} +# +# - name: Compute image tag +# id: tag +# run: | +# TAG="${{ inputs.image_tag }}" +# if [ -z "$TAG" ]; then +# TAG="sha-$(echo '${{ github.sha }}' | cut -c1-7)" +# fi +# echo "tag=${TAG}" >> "$GITHUB_OUTPUT" +# echo "Publishing to ghcr.io/vsco → ${REGISTRY}/${IMAGE_ORG}/${IMAGE_NAME}:${TAG}" +# +# - name: Build and push ome-manager (linux/amd64) +# uses: docker/build-push-action@v7 +# with: +# context: . +# file: dockerfiles/manager.Dockerfile +# platforms: linux/amd64 +# push: true +# tags: | +# ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} +# cache-from: type=gha +# cache-to: type=gha,mode=max +# build-args: | +# VERSION=${{ steps.tag.outputs.tag }} +# GIT_TAG=${{ steps.tag.outputs.tag }} +# GIT_COMMIT=${{ github.sha }} +# +# - name: Summary +# run: | +# echo "## VSCO ome-manager image" >> "$GITHUB_STEP_SUMMARY" +# echo "" >> "$GITHUB_STEP_SUMMARY" +# echo "Pushed: \`${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}\`" >> "$GITHUB_STEP_SUMMARY" +# echo "" >> "$GITHUB_STEP_SUMMARY" +# echo "Remove \`.github/workflows/vsco-publish-ome-manager.yaml\` before submitting a PR to sgl-project/ome." >> "$GITHUB_STEP_SUMMARY" From a8a009bbb3e4b63410d08c654caea56a648cc614 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 2 Apr 2026 19:28:59 -0600 Subject: [PATCH 18/35] Update the chart to pull images from vsco instead of the other --- charts/ome-resources/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index 50d522dd3..f67f6ab09 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -1,12 +1,12 @@ # VSCO fork: Helm charts are published to oci://ghcr.io/vsco/charts (see .github/workflows/vsco-publish-helm-charts.yaml). # shared-infra only sets var.ome_version (chart semver). Forked ome-manager uses a full image ref so other images can stay on global.hub. global: - hub: "ghcr.io/moirai-internal" + hub: "ghcr.io/vsco" imagePullSecrets: - name: ghcr-creds ome: - version: &defaultVersion v0.1.4-vsco1 + version: &defaultVersion v0.1.4 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" @@ -54,7 +54,7 @@ ome: topologySpreadConstraints: [] affinity: {} # Full path → ome.imageWithHub ignores global.hub (VSCO-built controller) - image: ghcr.io/vsco/ome-manager + image: ome-manager tag: *defaultVersion resources: limits: From 9891dd68a6774c28669a881808f15d46d2392922 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 2 Apr 2026 19:29:48 -0600 Subject: [PATCH 19/35] Missed an comment that didn't need to be there anymore --- charts/ome-resources/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index f67f6ab09..a2facc8cd 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -53,7 +53,6 @@ ome: tolerations: [] topologySpreadConstraints: [] affinity: {} - # Full path → ome.imageWithHub ignores global.hub (VSCO-built controller) image: ome-manager tag: *defaultVersion resources: From 0d649f68ae15048bd73c3196c8c5449adc6b3ae8 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Fri, 10 Apr 2026 10:59:49 -0700 Subject: [PATCH 20/35] add ingressclassname variable for overriding --- .../workflows/vsco-publish-ome-manager.yaml | 4 ++-- charts/ome-resources/values.yaml | 2 +- pkg/constants/constants.go | 1 + .../inferenceservice/utils/annotations.go | 5 ++++ .../utils/annotations_test.go | 23 +++++++++++++++++++ .../content/en/docs/administration/ingress.md | 1 + 6 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 7c0124ba2..1a616e7a4 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.4-vsco2. Override "image_tag" +## Default tag matches local manual pushes: v0.1.4-vsco3. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.4-vsco2' +# default: 'v0.1.4-vsco3' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index a2facc8cd..7257f2943 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.4 + version: &defaultVersion v0.1.4-vsco3 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 498c4c89d..a2864d55d 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -135,6 +135,7 @@ var ( IngressPathTemplate = OMEAPIGroupName + "/ingress-path-template" IngressDisableIstioVirtualHost = OMEAPIGroupName + "/ingress-disable-istio-virtualhost" IngressDisableCreation = OMEAPIGroupName + "/ingress-disable-creation" + IngressClassName = OMEAPIGroupName + "/ingress-class-name" ) // InferenceService Annotations for model encryption and decryption diff --git a/pkg/controller/v1beta1/inferenceservice/utils/annotations.go b/pkg/controller/v1beta1/inferenceservice/utils/annotations.go index 5aea9ccae..756c5e702 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/annotations.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/annotations.go @@ -119,6 +119,11 @@ func ResolveIngressConfig(baseConfig *controllerconfig.IngressConfig, annotation resolved.IngressDomain = ingressDomain } + if className, exists := annotations[constants.IngressClassName]; exists && className != "" { + v := className + resolved.IngressClassName = &v + } + if urlScheme, exists := annotations[constants.IngressURLScheme]; exists && urlScheme != "" { resolved.UrlScheme = urlScheme } diff --git a/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go b/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go index 67b137e12..24d6534a5 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go @@ -9,6 +9,10 @@ import ( "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" ) +func strPtr(s string) *string { + return &s +} + func TestResolveIngressConfig(t *testing.T) { // Base config from ConfigMap baseConfig := &controllerconfig.IngressConfig{ @@ -115,11 +119,29 @@ func TestResolveIngressConfig(t *testing.T) { DisableIngressCreation: false, }, }, + { + name: "ingress class name override", + annotations: map[string]string{ + constants.IngressClassName: "private-ingress-controller", + }, + expected: &controllerconfig.IngressConfig{ + IngressGateway: "knative-serving/knative-ingress-gateway", + IngressServiceName: "istio-ingressgateway.istio-system.svc.cluster.local", + IngressDomain: "svc.cluster.local", + DomainTemplate: "{{ .Name }}.{{ .Namespace }}.{{ .IngressDomain }}", + UrlScheme: "http", + PathTemplate: "", + DisableIstioVirtualHost: false, + DisableIngressCreation: false, + IngressClassName: strPtr("private-ingress-controller"), + }, + }, { name: "comprehensive override", annotations: map[string]string{ constants.IngressDomainTemplate: "{{ .Name }}-prod.company.com", constants.IngressDomain: "company.com", + constants.IngressClassName: "nginx", constants.IngressURLScheme: "https", constants.IngressPathTemplate: "/ml/{{ .Name }}", constants.IngressAdditionalDomains: "backup.com,mirror.net", @@ -136,6 +158,7 @@ func TestResolveIngressConfig(t *testing.T) { DisableIstioVirtualHost: false, DisableIngressCreation: false, AdditionalIngressDomains: &[]string{"backup.com", "mirror.net"}, + IngressClassName: strPtr("nginx"), }, }, } diff --git a/site/content/en/docs/administration/ingress.md b/site/content/en/docs/administration/ingress.md index a863ea8a3..21a284e04 100644 --- a/site/content/en/docs/administration/ingress.md +++ b/site/content/en/docs/administration/ingress.md @@ -129,6 +129,7 @@ The following ingress settings can be overridden per service: |--------------------------------------------|----------------------------|---------------------------------|--------------------------| | `ome.io/ingress-domain-template` | `domainTemplate` | Custom domain pattern | String template | | `ome.io/ingress-domain` | `ingressDomain` | Fixed base domain | String | +| `ome.io/ingress-class-name` | `ingressClassName` | Ingress controller class | String | | `ome.io/ingress-additional-domains` | `additionalIngressDomains` | Extra domains (comma-separated) | String list | | `ome.io/ingress-url-scheme` | `urlScheme` | HTTP/HTTPS scheme | String | | `ome.io/ingress-path-template` | `pathTemplate` | URL path pattern | String template | From 1e82064e355fea9396cc16df2f2bf781615e5c4a Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Fri, 24 Apr 2026 14:31:02 -0700 Subject: [PATCH 21/35] add support for pvc annotation --- .../workflows/vsco-publish-helm-charts.yaml | 101 ++++++++++++++++++ .../model-agent-daemonset/daemonset.yaml | 6 ++ .../templates/model-agent-daemonset/pvc.yaml | 16 +++ .../templates/ome-controller/configmap.yaml | 2 + charts/ome-resources/values.yaml | 12 +++ pkg/constants/constants.go | 12 ++- .../v1beta1/controllerconfig/configmap.go | 21 +++- .../controllerconfig/configmap_test.go | 3 + .../inferenceservice/components/base.go | 24 ++++- .../inferenceservice/components/base_test.go | 64 +++++++++++ .../inferenceservice/utils/model_storage.go | 58 ++++++++++ .../utils/model_storage_test.go | 52 +++++++++ 12 files changed, 361 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/vsco-publish-helm-charts.yaml create mode 100644 charts/ome-resources/templates/model-agent-daemonset/pvc.yaml create mode 100644 pkg/controller/v1beta1/inferenceservice/utils/model_storage.go create mode 100644 pkg/controller/v1beta1/inferenceservice/utils/model_storage_test.go diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml new file mode 100644 index 000000000..7980c4cb3 --- /dev/null +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -0,0 +1,101 @@ +# ============================================================================= +# VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO sgl-project/ome +# +# Packages ome-crd, ome-resources, ome-serving and pushes OCI charts to: +# oci://ghcr.io/vsco/charts +# +# Use a Helm semver for chart_version (e.g. 0.1.4), not a leading "v". +# Point shared-infra at the same semver: var.ome_version = "0.1.4" and +# repository = "oci://ghcr.io/vsco/charts" +# in terraform/modules/eks/backend-services/ome.tf +# +# Auth: repo secret VSCO_GHCR_TOKEN (PAT write:packages) if GITHUB_TOKEN cannot push. +# ============================================================================= + +name: VSCO (fork) publish OME Helm charts to ghcr.io/vsco + +on: + workflow_dispatch: + inputs: + chart_version: + description: 'Helm chart semver (e.g. 0.1.4) — no leading v' + required: true + default: '0.1.4' + image_tag: + description: 'OME image tag written into values.yaml (e.g. v0.1.4-vsco3)' + required: true + default: 'v0.1.4-vsco3' + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + CHART_ORG: vsco + +jobs: + publish-charts: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Helm + uses: azure/setup-helm@v4 + with: + version: v3.16.0 + + - name: Install yq + run: | + curl -fsSL -X GET "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" -o /usr/local/bin/yq + chmod +x /usr/local/bin/yq + + - name: Log in to ghcr.io (org vsco packages) + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.VSCO_GHCR_TOKEN || secrets.GITHUB_TOKEN }} + + - name: Stamp chart and image versions + run: | + set -euo pipefail + VERSION="${{ inputs.chart_version }}" + TAG="${{ inputs.image_tag }}" + for chart in charts/*/; do + yq eval -i ".version = \"${VERSION}\"" "${chart}Chart.yaml" + yq eval -i ".appVersion = \"${TAG}\"" "${chart}Chart.yaml" + done + chart="charts/ome-resources/" + if [[ -f "${chart}values.yaml" ]]; then + yq eval -i ".ome.version = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".ome.benchmarkJob.tag = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".ome.multinodeProber.tag = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".ome.omeAgent.tag = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".modelAgent.image.tag = \"${TAG}\"" "${chart}values.yaml" + fi + + - name: Package and push charts + env: + CHART_VERSION: ${{ inputs.chart_version }} + run: | + set -euo pipefail + mkdir -p .charts-out + for chart in charts/*/; do + helm package "${chart}" -d .charts-out + done + for tgz in .charts-out/*.tgz; do + helm push "${tgz}" "oci://${{ env.REGISTRY }}/${{ env.CHART_ORG }}/charts" + done + echo "Published charts at oci://${{ env.REGISTRY }}/${{ env.CHART_ORG }}/charts (chart version ${CHART_VERSION})" + + - name: Summary + run: | + echo "## VSCO OME Helm charts" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "OCI registry: \`oci://${{ env.REGISTRY }}/${{ env.CHART_ORG }}/charts\`" >> "$GITHUB_STEP_SUMMARY" + echo "Chart semver: \`${{ inputs.chart_version }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "Image tag in ome-resources values: \`${{ inputs.image_tag }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Set \`var.ome_version\` to the chart semver and \`repository\` to this OCI URL in shared-infra \`ome.tf\`." >> "$GITHUB_STEP_SUMMARY" diff --git a/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml b/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml index 4bcadcef6..833fa933d 100644 --- a/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml +++ b/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml @@ -38,10 +38,16 @@ spec: {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} volumes: +{{- if .Values.modelAgent.persistence.enabled }} + - name: host-models + persistentVolumeClaim: + claimName: {{ .Values.modelAgent.persistence.claimName | quote }} +{{- else }} - name: host-models hostPath: path: {{ .Values.modelAgent.hostPath }} type: DirectoryOrCreate +{{- end }} {{- with .Values.modelAgent.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/ome-resources/templates/model-agent-daemonset/pvc.yaml b/charts/ome-resources/templates/model-agent-daemonset/pvc.yaml new file mode 100644 index 000000000..c85dfcce8 --- /dev/null +++ b/charts/ome-resources/templates/model-agent-daemonset/pvc.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.modelAgent.persistence.enabled .Values.modelAgent.persistence.create }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.modelAgent.persistence.claimName }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + {{- range .Values.modelAgent.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.modelAgent.persistence.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.modelAgent.persistence.size | quote }} +{{- end }} diff --git a/charts/ome-resources/templates/ome-controller/configmap.yaml b/charts/ome-resources/templates/ome-controller/configmap.yaml index 5c9253deb..c8f53bc0e 100644 --- a/charts/ome-resources/templates/ome-controller/configmap.yaml +++ b/charts/ome-resources/templates/ome-controller/configmap.yaml @@ -78,6 +78,8 @@ data: "scalingThreshold": "{{ .Values.ome.kedaConfig.scalingThreshold | default "10" }}", "scalingOperator": "{{ .Values.ome.kedaConfig.scalingOperator | default "GreaterThanOrEqual" }}" } + modelStorage: |- +{{ (.Values.ome.modelStorage | default dict) | toJson | nindent 4 }} --- apiVersion: v1 kind: ConfigMap diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index 7257f2943..d0c3f077a 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -87,8 +87,20 @@ ome: customPromQuery: "" scalingThreshold: "10" scalingOperator: "GreaterThanOrEqual" + # JSON merged into inferenceservice-config modelStorage (pvcClaimName / pvcMountRoot). + # Per-InferenceService override: annotation ome.io/model-storage-pvc. + modelStorage: {} modelAgent: hostPath: /mnt/data/models + # EFS (ReadWriteMany) or other RWX PVC for shared model cache; aligns with ome.modelStorage. + persistence: + enabled: false + create: false + claimName: ome-models-efs + accessModes: + - ReadWriteMany + storageClassName: efs-sc-gp + size: 200Gi priorityClassName: system-node-critical serviceAccountName: ome-model-agent image: diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index a2864d55d..12f9d6d70 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -339,7 +339,16 @@ const ( // SkipModelReadyNodeSelectorAnnotationKey, when set to "true" on an InferenceService, omits the // models.ome.io/...=Ready nodeSelector from engine and decoder pods. Use this with elastic GPU // pools (e.g. Karpenter) where nodes are provisioned before the model-agent applies the Ready label. -const SkipModelReadyNodeSelectorAnnotationKey = "ome.io/skip-model-ready-node-selector" +// +// ModelStoragePVCAnnotationKey names the PersistentVolumeClaim (in the InferenceService namespace) +// for base model files instead of hostPath; overrides ConfigMap modelStorage.pvcClaimName. +// DefaultModelPVCMountRoot must align with model-agent --models-root-dir and storage paths. +const ( + SkipModelReadyNodeSelectorAnnotationKey = "ome.io/skip-model-ready-node-selector" + // ModelStoragePVCAnnotationKey must stay in sync with OMEAPIGroupName + "/model-storage-pvc" (OMEAPIGroupName is a var). + ModelStoragePVCAnnotationKey = "ome.io/model-storage-pvc" + DefaultModelPVCMountRoot = "/mnt/data/models" +) // InferenceService default/canary constants const ( @@ -426,6 +435,7 @@ var ( ModelInitInjectionKey, // ome.io/inject-model-init - triggers model init container injection via webhook FineTunedAdapterInjectionKey, // ome.io/inject-fine-tuned-adapter - triggers fine-tuned adapter injection via webhook ServingSidecarInjectionKey, // ome.io/inject-serving-sidecar - triggers serving sidecar injection via webhook + ModelStoragePVCAnnotationKey, // ome.io/model-storage-pvc - per-ISVC PVC claim for base model files } ) diff --git a/pkg/controller/v1beta1/controllerconfig/configmap.go b/pkg/controller/v1beta1/controllerconfig/configmap.go index 44a727a54..ffd36e201 100644 --- a/pkg/controller/v1beta1/controllerconfig/configmap.go +++ b/pkg/controller/v1beta1/controllerconfig/configmap.go @@ -14,10 +14,11 @@ import ( ) const ( - IngressConfigKeyName = "ingress" - DeployConfigName = "deploy" - MultiNodeProberName = "multinodeProber" - BenchmarkJobConfigName = "benchmarkjob" + IngressConfigKeyName = "ingress" + DeployConfigName = "deploy" + MultiNodeProberName = "multinodeProber" + ModelStorageConfigName = "modelStorage" + BenchmarkJobConfigName = "benchmarkjob" DefaultDomainTemplate = "{{ .Name }}.{{ .Namespace }}.{{ .IngressDomain }}" DefaultIngressDomain = "example.com" @@ -44,10 +45,21 @@ type PodConfig struct { MemoryLimit string `json:"memoryLimit"` } +// +kubebuilder:object:generate=false +type ModelStorageConfig struct { + // PVCClaimName, when set, switches engine/decoder base-model volumes from hostPath to this + // PersistentVolumeClaim in the InferenceService namespace (ReadWriteMany, e.g. EFS). + PVCClaimName string `json:"pvcClaimName,omitempty"` + // PVCMountRoot is the path inside the pod that is the root of the PVC (SubPath is computed + // relative to this and the model Storage.Path). Defaults to /mnt/data/models. + PVCMountRoot string `json:"pvcMountRoot,omitempty"` +} + // +kubebuilder:object:generate=false type InferenceServicesConfig struct { // MultiNodeProber contains all MultiNodeProber Configuration MultiNodeProber MultiNodeProberConfig `json:"multinodeProber"` + ModelStorage ModelStorageConfig `json:"modelStorage,omitempty"` } // +kubebuilder:object:generate=false @@ -96,6 +108,7 @@ func NewInferenceServicesConfig(clientset kubernetes.Interface) (*InferenceServi icfg := &InferenceServicesConfig{} for _, err := range []error{ getComponentConfig(MultiNodeProberName, configMap, &icfg.MultiNodeProber), + getComponentConfig(ModelStorageConfigName, configMap, &icfg.ModelStorage), } { if err != nil { return nil, err diff --git a/pkg/controller/v1beta1/controllerconfig/configmap_test.go b/pkg/controller/v1beta1/controllerconfig/configmap_test.go index 0558ee5a3..04c4fae89 100644 --- a/pkg/controller/v1beta1/controllerconfig/configmap_test.go +++ b/pkg/controller/v1beta1/controllerconfig/configmap_test.go @@ -69,11 +69,14 @@ func TestNewInferenceServicesConfig(t *testing.T) { "startupTimeoutSeconds": 30, "unavailableThresholdSeconds": 60 }`, + ModelStorageConfigName: `{"pvcClaimName":"ome-models-efs","pvcMountRoot":"/mnt/data/models"}`, }, expectedError: false, validateConfig: func(t *testing.T, cfg *InferenceServicesConfig) { assert.Equal(t, "test-image", cfg.MultiNodeProber.Image) assert.Equal(t, "100m", cfg.MultiNodeProber.CPURequest) + assert.Equal(t, "ome-models-efs", cfg.ModelStorage.PVCClaimName) + assert.Equal(t, "/mnt/data/models", cfg.ModelStorage.PVCMountRoot) }, }, { diff --git a/pkg/controller/v1beta1/inferenceservice/components/base.go b/pkg/controller/v1beta1/inferenceservice/components/base.go index a457ba96a..9bbf280c5 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base.go @@ -96,6 +96,12 @@ func UpdateVolumeMounts(b *BaseComponentFields, isvc *v1beta1.InferenceService, MountPath: *b.BaseModel.Storage.Path, ReadOnly: true, } + if isvcutils.ModelStoragePVCClaimName(isvc, b.InferenceServiceConfig) != "" { + mountRoot := isvcutils.ModelStoragePVCMountRoot(b.InferenceServiceConfig) + if sub := isvcutils.ModelVolumeMountSubPathForPVC(mountRoot, *b.BaseModel.Storage.Path); sub != "" { + vm.SubPath = sub + } + } isvcutils.AppendVolumeMount(container, &vm) } } @@ -236,15 +242,23 @@ func UpdatePodSpecNodeSelector(b *BaseComponentFields, isvc *v1beta1.InferenceSe func UpdatePodSpecVolumes(b *BaseComponentFields, isvc *v1beta1.InferenceService, podSpec *corev1.PodSpec, objectMeta *metav1.ObjectMeta) { // Add model volume if base model is specified if b.BaseModel != nil && b.BaseModel.Storage != nil && b.BaseModel.Storage.Path != nil && b.BaseModelMeta != nil { - modelVolume := corev1.Volume{ - Name: b.BaseModelMeta.Name, - VolumeSource: corev1.VolumeSource{ + pvcClaim := isvcutils.ModelStoragePVCClaimName(isvc, b.InferenceServiceConfig) + modelVolume := corev1.Volume{Name: b.BaseModelMeta.Name} + if pvcClaim != "" { + modelVolume.VolumeSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcClaim, + ReadOnly: true, + }, + } + } else { + modelVolume.VolumeSource = corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: *b.BaseModel.Storage.Path, }, - }, + } } - podSpec.Volumes = append(podSpec.Volumes, modelVolume) + podSpec.Volumes = utils.AppendVolumeIfNotExists(podSpec.Volumes, modelVolume) } // Add empty model directory volume if required for fine-tuned serving diff --git a/pkg/controller/v1beta1/inferenceservice/components/base_test.go b/pkg/controller/v1beta1/inferenceservice/components/base_test.go index 289b80061..f9d7f28eb 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base_test.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base_test.go @@ -11,6 +11,7 @@ import ( "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" "github.com/sgl-project/ome/pkg/constants" + "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" ) func TestUpdatePodSpecNodeSelector(t *testing.T) { @@ -253,3 +254,66 @@ func TestProcessBaseLabels(t *testing.T) { g.Expect(labels).To(gomega.HaveKeyWithValue(constants.BaseModelTypeLabelKey, string(constants.ServingBaseModel))) g.Expect(labels).To(gomega.HaveKeyWithValue(constants.BaseModelVendorLabelKey, "meta")) } + +func TestUpdatePodSpecVolumes_PVCAndHostPath(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{ + Path: &modelPath, + }, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "cluster-model-x"}, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + + pod := &v1.PodSpec{} + UpdatePodSpecVolumes(b, isvc, pod, &metav1.ObjectMeta{}) + g.Expect(pod.Volumes).To(gomega.HaveLen(1)) + g.Expect(pod.Volumes[0].HostPath).NotTo(gomega.BeNil()) + g.Expect(pod.Volumes[0].HostPath.Path).To(gomega.Equal(modelPath)) + + b.InferenceServiceConfig = &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + } + pod2 := &v1.PodSpec{} + UpdatePodSpecVolumes(b, isvc, pod2, &metav1.ObjectMeta{}) + g.Expect(pod2.Volumes).To(gomega.HaveLen(1)) + g.Expect(pod2.Volumes[0].PersistentVolumeClaim).NotTo(gomega.BeNil()) + g.Expect(pod2.Volumes[0].PersistentVolumeClaim.ClaimName).To(gomega.Equal("ome-models-efs")) + g.Expect(pod2.Volumes[0].PersistentVolumeClaim.ReadOnly).To(gomega.BeTrue()) +} + +func TestUpdateVolumeMounts_PVCSubPath(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{ + Path: &modelPath, + }, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "cluster-model-x"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + container := &v1.Container{} + UpdateVolumeMounts(b, isvc, container, &metav1.ObjectMeta{Annotations: map[string]string{}}) + g.Expect(container.VolumeMounts).To(gomega.HaveLen(1)) + g.Expect(container.VolumeMounts[0].SubPath).To(gomega.Equal("my-model")) +} diff --git a/pkg/controller/v1beta1/inferenceservice/utils/model_storage.go b/pkg/controller/v1beta1/inferenceservice/utils/model_storage.go new file mode 100644 index 000000000..26822d5a4 --- /dev/null +++ b/pkg/controller/v1beta1/inferenceservice/utils/model_storage.go @@ -0,0 +1,58 @@ +package utils + +import ( + "path/filepath" + "strings" + + "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" + "github.com/sgl-project/ome/pkg/constants" + "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" +) + +// ModelStoragePVCClaimName returns the PVC claim name for base-model storage when using a +// shared filesystem (e.g. EFS ReadWriteMany). Order: InferenceService annotation +// (ome.io/model-storage-pvc), then cluster ConfigMap modelStorage.pvcClaimName, else empty +// (hostPath mode). +func ModelStoragePVCClaimName(isvc *v1beta1.InferenceService, cfg *controllerconfig.InferenceServicesConfig) string { + if isvc != nil { + if v := strings.TrimSpace(isvc.Annotations[constants.ModelStoragePVCAnnotationKey]); v != "" { + return v + } + } + if cfg != nil { + return strings.TrimSpace(cfg.ModelStorage.PVCClaimName) + } + return "" +} + +// ModelStoragePVCMountRoot returns the directory inside the pod that corresponds to the root of +// the shared PVC (used with SubPath so per-model paths align with hostPath layout). +func ModelStoragePVCMountRoot(cfg *controllerconfig.InferenceServicesConfig) string { + if cfg != nil && strings.TrimSpace(cfg.ModelStorage.PVCMountRoot) != "" { + return filepath.Clean(cfg.ModelStorage.PVCMountRoot) + } + return constants.DefaultModelPVCMountRoot +} + +// ModelVolumeMountSubPathForPVC returns the SubPath for a volumeMount when the model directory +// (storagePath) is under pvcMountRoot on the shared PVC. Empty means mount the whole claim at +// storagePath (single-directory layout). +func ModelVolumeMountSubPathForPVC(pvcMountRoot, storagePath string) string { + if pvcMountRoot == "" || storagePath == "" { + return "" + } + root := filepath.Clean(pvcMountRoot) + path := filepath.Clean(storagePath) + if root == path { + return "" + } + sep := string(filepath.Separator) + if path != root && !strings.HasPrefix(path, root+sep) { + return "" + } + rel, err := filepath.Rel(root, path) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return "" + } + return rel +} diff --git a/pkg/controller/v1beta1/inferenceservice/utils/model_storage_test.go b/pkg/controller/v1beta1/inferenceservice/utils/model_storage_test.go new file mode 100644 index 000000000..4cfdf9ad0 --- /dev/null +++ b/pkg/controller/v1beta1/inferenceservice/utils/model_storage_test.go @@ -0,0 +1,52 @@ +package utils + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" + "github.com/sgl-project/ome/pkg/constants" + "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" +) + +func TestModelVolumeMountSubPathForPVC(t *testing.T) { + root := constants.DefaultModelPVCMountRoot + nested := filepath.Join(root, "a", "b") + tests := []struct { + name string + mountRoot string + storagePath string + want string + }{ + {"equal root", root, root, ""}, + {"subdir", root, filepath.Join(root, "qwen3-vl-8b-instruct"), "qwen3-vl-8b-instruct"}, + {"nested", root, nested, filepath.Join("a", "b")}, + {"outside tree", root, "/other/path", ""}, + {"empty root", "", filepath.Join(root, "x"), ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ModelVolumeMountSubPathForPVC(tt.mountRoot, tt.storagePath) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestModelStoragePVCClaimName(t *testing.T) { + cfg := &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{PVCClaimName: "from-config"}, + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{constants.ModelStoragePVCAnnotationKey: " from-annotation "}, + }, + } + assert.Equal(t, "from-annotation", ModelStoragePVCClaimName(isvc, cfg)) + + isvc2 := &v1beta1.InferenceService{ObjectMeta: metav1.ObjectMeta{}} + assert.Equal(t, "from-config", ModelStoragePVCClaimName(isvc2, cfg)) + assert.Equal(t, "", ModelStoragePVCClaimName(isvc2, nil)) +} From 16891d568edc39230356f7d74c6072e45d9abcbc Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Fri, 24 Apr 2026 14:42:48 -0700 Subject: [PATCH 22/35] revert: add ingressclassname variable for overriding; increment to v0.1.4-vsco4 for pvc annotation addition --- .../workflows/vsco-publish-ome-manager.yaml | 4 ++-- charts/ome-resources/values.yaml | 2 +- pkg/constants/constants.go | 1 - .../inferenceservice/utils/annotations.go | 5 ---- .../utils/annotations_test.go | 23 ------------------- .../content/en/docs/administration/ingress.md | 1 - 6 files changed, 3 insertions(+), 33 deletions(-) diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 1a616e7a4..1d15522ad 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.4-vsco3. Override "image_tag" +## Default tag matches local manual pushes: v0.1.4-vsco4. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.4-vsco3' +# default: 'v0.1.4-vsco4' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index d0c3f077a..0a46465f7 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.4-vsco3 + version: &defaultVersion v0.1.4-vsco4 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 12f9d6d70..bc10d8eae 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -135,7 +135,6 @@ var ( IngressPathTemplate = OMEAPIGroupName + "/ingress-path-template" IngressDisableIstioVirtualHost = OMEAPIGroupName + "/ingress-disable-istio-virtualhost" IngressDisableCreation = OMEAPIGroupName + "/ingress-disable-creation" - IngressClassName = OMEAPIGroupName + "/ingress-class-name" ) // InferenceService Annotations for model encryption and decryption diff --git a/pkg/controller/v1beta1/inferenceservice/utils/annotations.go b/pkg/controller/v1beta1/inferenceservice/utils/annotations.go index 756c5e702..5aea9ccae 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/annotations.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/annotations.go @@ -119,11 +119,6 @@ func ResolveIngressConfig(baseConfig *controllerconfig.IngressConfig, annotation resolved.IngressDomain = ingressDomain } - if className, exists := annotations[constants.IngressClassName]; exists && className != "" { - v := className - resolved.IngressClassName = &v - } - if urlScheme, exists := annotations[constants.IngressURLScheme]; exists && urlScheme != "" { resolved.UrlScheme = urlScheme } diff --git a/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go b/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go index 24d6534a5..67b137e12 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/annotations_test.go @@ -9,10 +9,6 @@ import ( "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" ) -func strPtr(s string) *string { - return &s -} - func TestResolveIngressConfig(t *testing.T) { // Base config from ConfigMap baseConfig := &controllerconfig.IngressConfig{ @@ -119,29 +115,11 @@ func TestResolveIngressConfig(t *testing.T) { DisableIngressCreation: false, }, }, - { - name: "ingress class name override", - annotations: map[string]string{ - constants.IngressClassName: "private-ingress-controller", - }, - expected: &controllerconfig.IngressConfig{ - IngressGateway: "knative-serving/knative-ingress-gateway", - IngressServiceName: "istio-ingressgateway.istio-system.svc.cluster.local", - IngressDomain: "svc.cluster.local", - DomainTemplate: "{{ .Name }}.{{ .Namespace }}.{{ .IngressDomain }}", - UrlScheme: "http", - PathTemplate: "", - DisableIstioVirtualHost: false, - DisableIngressCreation: false, - IngressClassName: strPtr("private-ingress-controller"), - }, - }, { name: "comprehensive override", annotations: map[string]string{ constants.IngressDomainTemplate: "{{ .Name }}-prod.company.com", constants.IngressDomain: "company.com", - constants.IngressClassName: "nginx", constants.IngressURLScheme: "https", constants.IngressPathTemplate: "/ml/{{ .Name }}", constants.IngressAdditionalDomains: "backup.com,mirror.net", @@ -158,7 +136,6 @@ func TestResolveIngressConfig(t *testing.T) { DisableIstioVirtualHost: false, DisableIngressCreation: false, AdditionalIngressDomains: &[]string{"backup.com", "mirror.net"}, - IngressClassName: strPtr("nginx"), }, }, } diff --git a/site/content/en/docs/administration/ingress.md b/site/content/en/docs/administration/ingress.md index 21a284e04..a863ea8a3 100644 --- a/site/content/en/docs/administration/ingress.md +++ b/site/content/en/docs/administration/ingress.md @@ -129,7 +129,6 @@ The following ingress settings can be overridden per service: |--------------------------------------------|----------------------------|---------------------------------|--------------------------| | `ome.io/ingress-domain-template` | `domainTemplate` | Custom domain pattern | String template | | `ome.io/ingress-domain` | `ingressDomain` | Fixed base domain | String | -| `ome.io/ingress-class-name` | `ingressClassName` | Ingress controller class | String | | `ome.io/ingress-additional-domains` | `additionalIngressDomains` | Extra domains (comma-separated) | String list | | `ome.io/ingress-url-scheme` | `urlScheme` | HTTP/HTTPS scheme | String | | `ome.io/ingress-path-template` | `pathTemplate` | URL path pattern | String template | From 935ceb7be8ae255d781d9b0eb0dbc5bfc7651489 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Mon, 27 Apr 2026 15:23:24 -0700 Subject: [PATCH 23/35] add support for fine tuned weights for PVC backed adapters --- .../inferenceservice/components/base.go | 24 ++++-- .../components/engine_test.go | 60 +++++++++++++ .../pod/fine_tuned_adapter_injector.go | 10 ++- .../pod/fine_tuned_adapter_injector_test.go | 85 +++++++++++++++++++ 4 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go diff --git a/pkg/controller/v1beta1/inferenceservice/components/base.go b/pkg/controller/v1beta1/inferenceservice/components/base.go index 9bbf280c5..68d131abe 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" "strconv" + "strings" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -19,6 +20,7 @@ import ( "github.com/sgl-project/ome/pkg/controller/v1beta1/inferenceservice/status" isvcutils "github.com/sgl-project/ome/pkg/controller/v1beta1/inferenceservice/utils" "github.com/sgl-project/ome/pkg/utils" + "github.com/sgl-project/ome/pkg/utils/storage" ) // BaseComponentFields contains common fields for all components @@ -392,16 +394,26 @@ func UpdateDecoderAffinity(b *BaseComponentFields, isvc *v1beta1.InferenceServic func ProcessBaseAnnotations(b *BaseComponentFields, isvc *v1beta1.InferenceService, annotations map[string]string) (map[string]string, error) { // Add fine-tuned weight annotations if applicable if b.FineTunedServing && len(b.FineTunedWeights) > 0 { - // Inject ft adapter for single/non-stacked fine-tuned weight downloading - annotations[constants.FineTunedAdapterInjectionKey] = b.FineTunedWeights[0].Name - - // Add fine-tuned weight ft strategy - fineTunedWeightFTStrategy, err := isvcutils.GetValueFromRawExtension(b.FineTunedWeights[0].Spec.HyperParameters, constants.StrategyConfigKey) + ftw := b.FineTunedWeights[0] + fineTunedWeightFTStrategy, err := isvcutils.GetValueFromRawExtension(ftw.Spec.HyperParameters, constants.StrategyConfigKey) if err != nil || fineTunedWeightFTStrategy == nil { - b.Log.Error(err, "Error getting hyper-parameter strategy from FineTunedWeight", "FineTunedWeight", b.FineTunedWeights[0].Name, "namespace", isvc.Namespace) + b.Log.Error(err, "Error getting hyper-parameter strategy from FineTunedWeight", "FineTunedWeight", ftw.Name, "namespace", isvc.Namespace) return nil, err } annotations[constants.FineTunedWeightFTStrategyKey] = fineTunedWeightFTStrategy.(string) + + uri := "" + if ftw.Spec.Storage != nil && ftw.Spec.Storage.StorageUri != nil { + uri = strings.TrimSpace(*ftw.Spec.Storage.StorageUri) + } + // OCI/S3 fine-tuned adapter init expects object storage URIs. PVC-backed weights (e.g. Git LFS + // materialized onto a shared volume) are already on disk; skip injection. + if strings.HasPrefix(uri, storage.PVCStoragePrefix) { + b.Log.Info("Skipping fine-tuned adapter injection for PVC-backed FineTunedWeight", + "FineTunedWeight", ftw.Name, "namespace", isvc.Namespace) + } else { + annotations[constants.FineTunedAdapterInjectionKey] = ftw.Name + } } if b.FineTunedServingWithMergedWeights { diff --git a/pkg/controller/v1beta1/inferenceservice/components/engine_test.go b/pkg/controller/v1beta1/inferenceservice/components/engine_test.go index 4fd0fe40b..751683b33 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/engine_test.go +++ b/pkg/controller/v1beta1/inferenceservice/components/engine_test.go @@ -743,6 +743,7 @@ func TestEngineReconcileObjectMeta(t *testing.T) { fineTunedServing bool fineTunedWeights []*v1beta1.FineTunedWeight expectedAnnotations map[string]string + annotationAbsences []string expectedLabels map[string]string expectedName string }{ @@ -862,6 +863,62 @@ func TestEngineReconcileObjectMeta(t *testing.T) { }, expectedName: "ft-isvc-engine", }, + { + name: "Fine-tuned serving metadata with PVC-backed adapter", + isvc: &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ft-pvc-isvc", + Namespace: "default", + }, + }, + engineSpec: &v1beta1.EngineSpec{}, + baseModel: &v1beta1.BaseModelSpec{ + ModelFormat: v1beta1.ModelFormat{ + Name: "safetensors", + }, + ModelExtensionSpec: v1beta1.ModelExtensionSpec{ + Vendor: stringPtr("meta"), + }, + }, + baseModelMeta: &metav1.ObjectMeta{ + Name: "llama-base", + }, + fineTunedServing: true, + fineTunedWeights: []*v1beta1.FineTunedWeight{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ft-weight-pvc", + }, + Spec: v1beta1.FineTunedWeightSpec{ + HyperParameters: runtime.RawExtension{ + Raw: []byte(`{"strategy": "lora"}`), + }, + Storage: &v1beta1.StorageSpec{ + StorageUri: stringPtr("pvc://ome-models-efs/adapters/qwen3-vl-8b-lora"), + }, + }, + }, + }, + expectedAnnotations: map[string]string{ + constants.FineTunedWeightFTStrategyKey: "lora", + constants.BaseModelName: "llama-base", + constants.BaseModelFormat: "safetensors", + constants.BaseModelVendorAnnotationKey: "meta", + }, + annotationAbsences: []string{constants.FineTunedAdapterInjectionKey}, + expectedLabels: map[string]string{ + constants.InferenceServicePodLabelKey: "ft-pvc-isvc", + constants.OMEComponentLabel: "engine", + constants.FTServingLabelKey: "true", + constants.FineTunedWeightFTStrategyLabelKey: "lora", + constants.FTServingWithMergedWeightsLabelKey: "false", + constants.InferenceServiceBaseModelNameLabelKey: "llama-base", + constants.InferenceServiceBaseModelSizeLabelKey: "SMALL", + constants.BaseModelTypeLabelKey: "Serving", + constants.BaseModelVendorLabelKey: "meta", + }, + expectedName: "ft-pvc-isvc-engine", + }, } for _, tt := range tests { @@ -908,6 +965,9 @@ func TestEngineReconcileObjectMeta(t *testing.T) { for k, v := range tt.expectedAnnotations { g.Expect(objectMeta.Annotations).To(gomega.HaveKeyWithValue(k, v)) } + for _, k := range tt.annotationAbsences { + g.Expect(objectMeta.Annotations).NotTo(gomega.HaveKey(k)) + } // Validate labels for k, v := range tt.expectedLabels { diff --git a/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go b/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go index a5cb59fd9..941896df8 100644 --- a/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go +++ b/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go @@ -3,6 +3,7 @@ package pod import ( "encoding/json" "fmt" + "strings" "github.com/go-playground/validator/v10" v1 "k8s.io/api/core/v1" @@ -47,7 +48,14 @@ func newFineTunedAdapterInjector(configMap *v1.ConfigMap, client client.Client) // InjectFineTunedAdapter injects the fine-tuned weight initialization container into the pod if necessary. func (fa *FineTunedAdapterInjector) InjectFineTunedAdapter(pod *v1.Pod) error { if fineTunedWeightName, ok := pod.ObjectMeta.Annotations[constants.FineTunedAdapterInjectionKey]; ok && len(fineTunedWeightName) > 0 { - // set the fine-tuned weight name + // OCI adapter init is not used for PVC-backed weights (already on disk). Skip before validate() + // so clusters without fineTunedAdapter ConfigMap (Image/CompartmentId/…) do not deny pod admission. + ftw, err := isvcutils.GetFineTunedWeight(fa.client, fineTunedWeightName) + if err == nil && ftw.Spec.Storage != nil && ftw.Spec.Storage.StorageUri != nil { + if strings.HasPrefix(strings.TrimSpace(*ftw.Spec.Storage.StorageUri), storage.PVCStoragePrefix) { + return nil + } + } fa.fineTunedWeightName = fineTunedWeightName return fa.injectFineTunedAdapter(pod) } diff --git a/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go b/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go new file mode 100644 index 000000000..06d922e0d --- /dev/null +++ b/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go @@ -0,0 +1,85 @@ +package pod + +import ( + "context" + "testing" + + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlclientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" + "github.com/sgl-project/ome/pkg/constants" +) + +func TestInjectFineTunedAdapter_SkipsWhenFineTunedWeightIsPVC(t *testing.T) { + g := gomega.NewGomegaWithT(t) + scheme := runtime.NewScheme() + g.Expect(v1beta1.AddToScheme(scheme)).To(gomega.Succeed()) + + pvcURI := "pvc://ome-models-efs/adapters/qwen3-vl-8b-lora" + ftw := &v1beta1.FineTunedWeight{ + ObjectMeta: metav1.ObjectMeta{Name: "qwen3-vl-8b-lora"}, + Spec: v1beta1.FineTunedWeightSpec{ + Storage: &v1beta1.StorageSpec{ + StorageUri: &pvcURI, + }, + }, + } + cl := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(ftw).Build() + + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "dummy", Namespace: constants.OMENamespace}, + Data: map[string]string{fineTunedAdapterConfigMapKeyName: "{}"}, + } + fa := newFineTunedAdapterInjector(cm, cl) + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.FineTunedAdapterInjectionKey: "qwen3-vl-8b-lora", + }, + }, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: constants.MainContainerName}}}, + } + g.Expect(fa.InjectFineTunedAdapter(pod)).To(gomega.Succeed()) + g.Expect(pod.Spec.InitContainers).To(gomega.BeEmpty()) +} + +func TestInjectFineTunedAdapter_RequiresOCIConfigWhenNotPVC(t *testing.T) { + g := gomega.NewGomegaWithT(t) + scheme := runtime.NewScheme() + g.Expect(v1beta1.AddToScheme(scheme)).To(gomega.Succeed()) + + s3URI := "s3://bucket/prefix" + ftw := &v1beta1.FineTunedWeight{ + ObjectMeta: metav1.ObjectMeta{Name: "ft-s3"}, + Spec: v1beta1.FineTunedWeightSpec{ + Storage: &v1beta1.StorageSpec{ + StorageUri: &s3URI, + }, + }, + } + cl := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(ftw).Build() + + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "dummy", Namespace: constants.OMENamespace}, + Data: map[string]string{fineTunedAdapterConfigMapKeyName: "{}"}, + } + fa := newFineTunedAdapterInjector(cm, cl) + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.FineTunedAdapterInjectionKey: "ft-s3", + }, + }, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: constants.MainContainerName}}}, + } + err := fa.InjectFineTunedAdapter(pod) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err.Error()).To(gomega.ContainSubstring("FineTunedAdapterInjector")) +} From 3393c2c7a3d20c0bf951144ac39c1a2db6df902c Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Mon, 27 Apr 2026 15:25:32 -0700 Subject: [PATCH 24/35] bump to version 0.1.4-vsco5 --- .../workflows/vsco-publish-helm-charts.yaml | 4 +- .../workflows/vsco-publish-ome-manager.yaml | 4 +- charts/ome-resources/values.yaml | 2 +- .../pod/fine_tuned_adapter_injector_test.go | 85 ------------------- 4 files changed, 5 insertions(+), 90 deletions(-) delete mode 100644 pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index 7980c4cb3..7fc331efc 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -22,9 +22,9 @@ on: required: true default: '0.1.4' image_tag: - description: 'OME image tag written into values.yaml (e.g. v0.1.4-vsco3)' + description: 'OME image tag written into values.yaml (e.g. v0.1.4-vsco5)' required: true - default: 'v0.1.4-vsco3' + default: 'v0.1.4-vsco5' permissions: contents: read diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 1d15522ad..3635b9663 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.4-vsco4. Override "image_tag" +## Default tag matches local manual pushes: v0.1.4-vsco5. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.4-vsco4' +# default: 'v0.1.4-vsco5' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index 0a46465f7..317e4a8ee 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.4-vsco4 + version: &defaultVersion v0.1.4-vsco5 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" diff --git a/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go b/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go deleted file mode 100644 index 06d922e0d..000000000 --- a/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package pod - -import ( - "context" - "testing" - - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - ctrlclientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" - "github.com/sgl-project/ome/pkg/constants" -) - -func TestInjectFineTunedAdapter_SkipsWhenFineTunedWeightIsPVC(t *testing.T) { - g := gomega.NewGomegaWithT(t) - scheme := runtime.NewScheme() - g.Expect(v1beta1.AddToScheme(scheme)).To(gomega.Succeed()) - - pvcURI := "pvc://ome-models-efs/adapters/qwen3-vl-8b-lora" - ftw := &v1beta1.FineTunedWeight{ - ObjectMeta: metav1.ObjectMeta{Name: "qwen3-vl-8b-lora"}, - Spec: v1beta1.FineTunedWeightSpec{ - Storage: &v1beta1.StorageSpec{ - StorageUri: &pvcURI, - }, - }, - } - cl := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(ftw).Build() - - cm := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "dummy", Namespace: constants.OMENamespace}, - Data: map[string]string{fineTunedAdapterConfigMapKeyName: "{}"}, - } - fa := newFineTunedAdapterInjector(cm, cl) - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - constants.FineTunedAdapterInjectionKey: "qwen3-vl-8b-lora", - }, - }, - Spec: v1.PodSpec{Containers: []v1.Container{{Name: constants.MainContainerName}}}, - } - g.Expect(fa.InjectFineTunedAdapter(pod)).To(gomega.Succeed()) - g.Expect(pod.Spec.InitContainers).To(gomega.BeEmpty()) -} - -func TestInjectFineTunedAdapter_RequiresOCIConfigWhenNotPVC(t *testing.T) { - g := gomega.NewGomegaWithT(t) - scheme := runtime.NewScheme() - g.Expect(v1beta1.AddToScheme(scheme)).To(gomega.Succeed()) - - s3URI := "s3://bucket/prefix" - ftw := &v1beta1.FineTunedWeight{ - ObjectMeta: metav1.ObjectMeta{Name: "ft-s3"}, - Spec: v1beta1.FineTunedWeightSpec{ - Storage: &v1beta1.StorageSpec{ - StorageUri: &s3URI, - }, - }, - } - cl := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(ftw).Build() - - cm := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "dummy", Namespace: constants.OMENamespace}, - Data: map[string]string{fineTunedAdapterConfigMapKeyName: "{}"}, - } - fa := newFineTunedAdapterInjector(cm, cl) - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - constants.FineTunedAdapterInjectionKey: "ft-s3", - }, - }, - Spec: v1.PodSpec{Containers: []v1.Container{{Name: constants.MainContainerName}}}, - } - err := fa.InjectFineTunedAdapter(pod) - g.Expect(err).To(gomega.HaveOccurred()) - g.Expect(err.Error()).To(gomega.ContainSubstring("FineTunedAdapterInjector")) -} From 8012ea9bbb917c7026b36b23392bab57a3140916 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 28 Apr 2026 05:04:28 -0700 Subject: [PATCH 25/35] bump to version 0.1.5-vsco1 to accommodate for 0.1.5 upstream rebase --- .github/workflows/vsco-publish-helm-charts.yaml | 12 ++++++------ .github/workflows/vsco-publish-ome-manager.yaml | 4 ++-- charts/ome-resources/values.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index 7fc331efc..270d91dc6 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -4,8 +4,8 @@ # Packages ome-crd, ome-resources, ome-serving and pushes OCI charts to: # oci://ghcr.io/vsco/charts # -# Use a Helm semver for chart_version (e.g. 0.1.4), not a leading "v". -# Point shared-infra at the same semver: var.ome_version = "0.1.4" and +# Use a Helm semver for chart_version (e.g. 0.1.5), not a leading "v". +# Point shared-infra at the same semver: var.ome_version = "0.1.5" and # repository = "oci://ghcr.io/vsco/charts" # in terraform/modules/eks/backend-services/ome.tf # @@ -18,13 +18,13 @@ on: workflow_dispatch: inputs: chart_version: - description: 'Helm chart semver (e.g. 0.1.4) — no leading v' + description: 'Helm chart semver (e.g. 0.1.5) — no leading v' required: true - default: '0.1.4' + default: '0.1.5' image_tag: - description: 'OME image tag written into values.yaml (e.g. v0.1.4-vsco5)' + description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco1)' required: true - default: 'v0.1.4-vsco5' + default: 'v0.1.5-vsco1' permissions: contents: read diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 3635b9663..c6dd06e18 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.4-vsco5. Override "image_tag" +## Default tag matches local manual pushes: v0.1.5-vsco1. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.4-vsco5' +# default: 'v0.1.5-vsco1' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index 317e4a8ee..abefadf24 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.4-vsco5 + version: &defaultVersion v0.1.5-vsco1 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" From 0d331e15e48c2d1b16bd1c24b95296528dd982d1 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 28 Apr 2026 06:39:41 -0700 Subject: [PATCH 26/35] remove arm64 building to reduce build times --- .github/workflows/dev-images.yaml | 5 +---- .github/workflows/release.yaml | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dev-images.yaml b/.github/workflows/dev-images.yaml index 4f03d4917..6b40d3617 100644 --- a/.github/workflows/dev-images.yaml +++ b/.github/workflows/dev-images.yaml @@ -46,9 +46,6 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -79,7 +76,7 @@ jobs: with: context: . file: ${{ matrix.component.dockerfile }} - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ matrix.component.image }}:dev diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d2851a069..0802e281b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -71,9 +71,6 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -93,7 +90,7 @@ jobs: with: context: . file: ${{ matrix.component.dockerfile }} - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ matrix.component.image }}:${{ needs.prepare.outputs.tag }} From bf1f3a8c641f21579c439a7d976417c6a88ba0d0 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 28 Apr 2026 07:08:53 -0700 Subject: [PATCH 27/35] fix updateStatus for using shared pvc for prefetched models --- .../workflows/vsco-publish-helm-charts.yaml | 4 +- .../workflows/vsco-publish-ome-manager.yaml | 4 +- charts/ome-resources/values.yaml | 2 +- .../v1beta1/inferenceservice/controller.go | 47 ++++++++++++------- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index 270d91dc6..9aa28ad5b 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -22,9 +22,9 @@ on: required: true default: '0.1.5' image_tag: - description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco1)' + description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco2)' required: true - default: 'v0.1.5-vsco1' + default: 'v0.1.5-vsco2' permissions: contents: read diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index c6dd06e18..ee98ec7b4 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.5-vsco1. Override "image_tag" +## Default tag matches local manual pushes: v0.1.5-vsco2. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.5-vsco1' +# default: 'v0.1.5-vsco2' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index abefadf24..e0dc44660 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.5-vsco1 + version: &defaultVersion v0.1.5-vsco2 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index 7da7bc32c..e0df65764 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" knapis "knative.dev/pkg/apis" duckv1 "knative.dev/pkg/apis/duck/v1" "knative.dev/pkg/network" @@ -556,24 +557,27 @@ func (r *InferenceServiceReconciler) handleServerlessPrerequisites(isvc *v1beta1 } func (r *InferenceServiceReconciler) updateStatus(desiredService *v1beta1.InferenceService, deploymentMode constants.DeploymentModeType) error { - existingService := &v1beta1.InferenceService{} - namespacedName := types.NamespacedName{Name: desiredService.Name, Namespace: desiredService.Namespace} - if err := r.Get(context.TODO(), namespacedName, existingService); err != nil { - return err - } - wasReady := inferenceServiceReadiness(existingService.Status) - if inferenceServiceStatusEqual(existingService.Status, desiredService.Status) { - // If we didn't change anything then don't call updateStatus. - // This is important because the copy we loaded from the informer's - // cache may be stale, and we don't want to overwrite a prior update - // to status with this stale state. - } else if err := r.Status().Update(context.TODO(), desiredService); err != nil { - r.Log.Error(err, "Failed to update InferenceService status", "InferenceService", desiredService.Name) - r.Recorder.Eventf(desiredService, v1.EventTypeWarning, "UpdateFailed", - "Failed to update status for InferenceService %q: %v", desiredService.Name, err) - return errors.Wrapf(err, "fails to update InferenceService status") - } else { - // If there was a difference and there was no error. + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + existingService := &v1beta1.InferenceService{} + namespacedName := types.NamespacedName{Name: desiredService.Name, Namespace: desiredService.Namespace} + if err := r.Get(context.TODO(), namespacedName, existingService); err != nil { + return err + } + wasReady := inferenceServiceReadiness(existingService.Status) + if inferenceServiceStatusEqual(existingService.Status, desiredService.Status) { + // If we didn't change anything then don't call updateStatus. + // This is important because the copy we loaded from the informer's + // cache may be stale, and we don't want to overwrite a prior update + // to status with this stale state. + return nil + } + // Reconcile queue copy of isvc carries a stale ResourceVersion; status update must use + // the version from the server (same as existingService) or writes fail with 409 when + // Helm/Flux or other controllers bump metadata between reconcile and Update. + desiredService.ResourceVersion = existingService.ResourceVersion + if err := r.Status().Update(context.TODO(), desiredService); err != nil { + return err + } isReady := inferenceServiceReadiness(desiredService.Status) if wasReady && !isReady { // Moved to NotReady State r.Recorder.Eventf(desiredService, v1.EventTypeWarning, string(InferenceServiceNotReadyState), @@ -582,6 +586,13 @@ func (r *InferenceServiceReconciler) updateStatus(desiredService *v1beta1.Infere r.Recorder.Eventf(desiredService, v1.EventTypeNormal, string(InferenceServiceReadyState), fmt.Sprintf("InferenceService [%v] is Ready", desiredService.GetName())) } + return nil + }) + if err != nil { + r.Log.Error(err, "Failed to update InferenceService status", "InferenceService", desiredService.Name) + r.Recorder.Eventf(desiredService, v1.EventTypeWarning, "UpdateFailed", + "Failed to update status for InferenceService %q: %v", desiredService.Name, err) + return errors.Wrapf(err, "fails to update InferenceService status") } return nil } From 83fbfa555374bdbf4d9d37bcf529444c5f353d7f Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 28 Apr 2026 07:42:54 -0700 Subject: [PATCH 28/35] replace status on the object returned by Get (server view of spec + metadata + resourceVersion) --- .../v1beta1/inferenceservice/controller.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index e0df65764..a1d7970d8 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -571,13 +571,15 @@ func (r *InferenceServiceReconciler) updateStatus(desiredService *v1beta1.Infere // to status with this stale state. return nil } - // Reconcile queue copy of isvc carries a stale ResourceVersion; status update must use - // the version from the server (same as existingService) or writes fail with 409 when - // Helm/Flux or other controllers bump metadata between reconcile and Update. - desiredService.ResourceVersion = existingService.ResourceVersion - if err := r.Status().Update(context.TODO(), desiredService); err != nil { + // Apply only status onto the object we just read from the API. desiredService may carry + // stale spec/metadata from the reconcile queue while Helm/Flux updated the live object; + // Status().Update with mismatched spec+RV can still fail with 409 or other conflicts. + existingService.Status = *desiredService.Status.DeepCopy() + if err := r.Status().Update(context.TODO(), existingService); err != nil { return err } + desiredService.Status = existingService.Status + desiredService.ResourceVersion = existingService.ResourceVersion isReady := inferenceServiceReadiness(desiredService.Status) if wasReady && !isReady { // Moved to NotReady State r.Recorder.Eventf(desiredService, v1.EventTypeWarning, string(InferenceServiceNotReadyState), From f46972280ae3283cf4fd89c91c4d4fd92d6f0510 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 28 Apr 2026 07:45:56 -0700 Subject: [PATCH 29/35] bump to version 0.1.5-vsco3 --- .github/workflows/vsco-publish-helm-charts.yaml | 12 ++++++------ .github/workflows/vsco-publish-ome-manager.yaml | 4 ++-- charts/ome-resources/values.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index 9aa28ad5b..db4bc5b42 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -4,8 +4,8 @@ # Packages ome-crd, ome-resources, ome-serving and pushes OCI charts to: # oci://ghcr.io/vsco/charts # -# Use a Helm semver for chart_version (e.g. 0.1.5), not a leading "v". -# Point shared-infra at the same semver: var.ome_version = "0.1.5" and +# Use a Helm semver for chart_version (e.g. 0.1.5-vsco3), not a leading "v" on chart_version. +# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco3" and # repository = "oci://ghcr.io/vsco/charts" # in terraform/modules/eks/backend-services/ome.tf # @@ -18,13 +18,13 @@ on: workflow_dispatch: inputs: chart_version: - description: 'Helm chart semver (e.g. 0.1.5) — no leading v' + description: 'Helm chart semver (e.g. 0.1.5-vsco3) — no leading v' required: true - default: '0.1.5' + default: '0.1.5-vsco3' image_tag: - description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco2)' + description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco3)' required: true - default: 'v0.1.5-vsco2' + default: 'v0.1.5-vsco3' permissions: contents: read diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index ee98ec7b4..e595126a6 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.5-vsco2. Override "image_tag" +## Default tag matches local manual pushes: v0.1.5-vsco3. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.5-vsco2' +# default: 'v0.1.5-vsco3' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index e0dc44660..1c17477ae 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.5-vsco2 + version: &defaultVersion v0.1.5-vsco3 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" From 4c09ee4d20d1db1fb9d410783dc6556910bac563 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Wed, 29 Apr 2026 14:32:54 -0700 Subject: [PATCH 30/35] Gate the fine-tuned /opt/ml/model (and Cohere TFew) mounts on the same condition as the volume --- .../inferenceservice/components/base.go | 6 +- .../inferenceservice/components/base_test.go | 66 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/pkg/controller/v1beta1/inferenceservice/components/base.go b/pkg/controller/v1beta1/inferenceservice/components/base.go index 68d131abe..5c9c960bc 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base.go @@ -108,8 +108,10 @@ func UpdateVolumeMounts(b *BaseComponentFields, isvc *v1beta1.InferenceService, } } - // Add fine-tuned serving volume mounts - if b.FineTunedServing { + // Add fine-tuned serving volume mounts for /opt/ml/model only when the emptyDir volume is + // required (OCI/S3 adapter init or model-init). PVC-backed FineTunedWeights skip inject + // annotations; UpdatePodSpecVolumes then omits model-empty-dir, so mounts must match. + if b.FineTunedServing && isvcutils.IsEmptyModelDirVolumeRequired(objectMeta.Annotations) { defaultModelVolumeMount := corev1.VolumeMount{ Name: constants.ModelEmptyDirVolumeName, MountPath: constants.ModelDefaultMountPath, diff --git a/pkg/controller/v1beta1/inferenceservice/components/base_test.go b/pkg/controller/v1beta1/inferenceservice/components/base_test.go index f9d7f28eb..42360388c 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base_test.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base_test.go @@ -317,3 +317,69 @@ func TestUpdateVolumeMounts_PVCSubPath(t *testing.T) { g.Expect(container.VolumeMounts).To(gomega.HaveLen(1)) g.Expect(container.VolumeMounts[0].SubPath).To(gomega.Equal("my-model")) } + +// Fine-tuned serving with PVC-backed weights skips inject annotations; emptyDir for /opt/ml/model +// must not be mounted unless adapter/model-init injection requires it (matches UpdatePodSpecVolumes). +func TestUpdateVolumeMounts_FTWithoutEmptyDirSkipsModelEmptyDirMount(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/qwen3-vl-8b-instruct" + b := &BaseComponentFields{ + FineTunedServing: true, + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "qwen3-vl-8b-instruct"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + ann := map[string]string{ + constants.BaseModelName: "qwen3-vl-8b-instruct", + } + container := &v1.Container{} + UpdateVolumeMounts(b, isvc, container, &metav1.ObjectMeta{Annotations: ann}) + g.Expect(container.VolumeMounts).To(gomega.HaveLen(1)) + g.Expect(container.VolumeMounts[0].Name).To(gomega.Equal("qwen3-vl-8b-instruct")) + for _, vm := range container.VolumeMounts { + g.Expect(vm.Name).NotTo(gomega.Equal(constants.ModelEmptyDirVolumeName)) + } +} + +func TestUpdateVolumeMounts_FTWithInjectAddsModelEmptyDirMount(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + b := &BaseComponentFields{ + FineTunedServing: true, + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "cluster-model-x"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + ann := map[string]string{ + constants.FineTunedAdapterInjectionKey: "some-adapter", + } + container := &v1.Container{} + UpdateVolumeMounts(b, isvc, container, &metav1.ObjectMeta{Annotations: ann}) + names := make([]string, 0, len(container.VolumeMounts)) + for _, vm := range container.VolumeMounts { + names = append(names, vm.Name) + } + g.Expect(names).To(gomega.ContainElement(constants.ModelEmptyDirVolumeName)) +} From f9705115d94fbf3d5f300cdb688050fed7bc7cc4 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Wed, 29 Apr 2026 14:35:45 -0700 Subject: [PATCH 31/35] bump to version 0.1.5-vsco5 --- .github/workflows/vsco-publish-helm-charts.yaml | 12 ++++++------ .github/workflows/vsco-publish-ome-manager.yaml | 4 ++-- charts/ome-resources/values.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index db4bc5b42..b5130ff01 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -4,8 +4,8 @@ # Packages ome-crd, ome-resources, ome-serving and pushes OCI charts to: # oci://ghcr.io/vsco/charts # -# Use a Helm semver for chart_version (e.g. 0.1.5-vsco3), not a leading "v" on chart_version. -# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco3" and +# Use a Helm semver for chart_version (e.g. 0.1.5-vsco5), not a leading "v" on chart_version. +# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco5" and # repository = "oci://ghcr.io/vsco/charts" # in terraform/modules/eks/backend-services/ome.tf # @@ -18,13 +18,13 @@ on: workflow_dispatch: inputs: chart_version: - description: 'Helm chart semver (e.g. 0.1.5-vsco3) — no leading v' + description: 'Helm chart semver (e.g. 0.1.5-vsco5) — no leading v' required: true - default: '0.1.5-vsco3' + default: '0.1.5-vsco5' image_tag: - description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco3)' + description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco5)' required: true - default: 'v0.1.5-vsco3' + default: 'v0.1.5-vsco5' permissions: contents: read diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index e595126a6..37cf0c5a8 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.5-vsco3. Override "image_tag" +## Default tag matches local manual pushes: v0.1.5-vsco5. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.5-vsco3' +# default: 'v0.1.5-vsco5' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index 1c17477ae..fc28cf504 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.5-vsco3 + version: &defaultVersion v0.1.5-vsco5 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" From 4f8dac4efdcc7a7c5ff7e23c55c84e332cce4695 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Wed, 29 Apr 2026 15:08:39 -0700 Subject: [PATCH 32/35] New helper UpdateInitContainerBaseModelVolumeMounts for conditionally setting PVC --- .../workflows/vsco-publish-helm-charts.yaml | 12 ++-- .../workflows/vsco-publish-ome-manager.yaml | 4 +- charts/ome-resources/values.yaml | 2 +- .../inferenceservice/components/base.go | 40 +++++++++++++ .../inferenceservice/components/base_test.go | 56 +++++++++++++++++++ .../inferenceservice/components/decoder.go | 2 + .../inferenceservice/components/engine.go | 2 + 7 files changed, 109 insertions(+), 9 deletions(-) diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index b5130ff01..786b210e7 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -4,8 +4,8 @@ # Packages ome-crd, ome-resources, ome-serving and pushes OCI charts to: # oci://ghcr.io/vsco/charts # -# Use a Helm semver for chart_version (e.g. 0.1.5-vsco5), not a leading "v" on chart_version. -# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco5" and +# Use a Helm semver for chart_version (e.g. 0.1.5-vsco6), not a leading "v" on chart_version. +# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco6" and # repository = "oci://ghcr.io/vsco/charts" # in terraform/modules/eks/backend-services/ome.tf # @@ -18,13 +18,13 @@ on: workflow_dispatch: inputs: chart_version: - description: 'Helm chart semver (e.g. 0.1.5-vsco5) — no leading v' + description: 'Helm chart semver (e.g. 0.1.5-vsco6) — no leading v' required: true - default: '0.1.5-vsco5' + default: '0.1.5-vsco6' image_tag: - description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco5)' + description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco6)' required: true - default: 'v0.1.5-vsco5' + default: 'v0.1.5-vsco6' permissions: contents: read diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 37cf0c5a8..5a3393873 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.5-vsco5. Override "image_tag" +## Default tag matches local manual pushes: v0.1.5-vsco6. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.5-vsco5' +# default: 'v0.1.5-vsco6' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index fc28cf504..9ac9cc69d 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.5-vsco5 + version: &defaultVersion v0.1.5-vsco6 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" diff --git a/pkg/controller/v1beta1/inferenceservice/components/base.go b/pkg/controller/v1beta1/inferenceservice/components/base.go index 5c9c960bc..5c4f78166 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base.go @@ -138,6 +138,46 @@ func UpdateVolumeMounts(b *BaseComponentFields, isvc *v1beta1.InferenceService, } } +// UpdateInitContainerBaseModelVolumeMounts sets the PVC SubPath on init container volume mounts +// that target the base model volume and mount path. ServingRuntime templates usually specify +// only name + mountPath; UpdateVolumeMounts adds SubPath for the main container. Without the +// same SubPath here, the full claim is mounted at storagePath and paths like +// ${storagePath}/config.json (used by wait-for-base-model) miss the per-model subdirectory that +// prefetch and model-agent use on the shared PVC. +func UpdateInitContainerBaseModelVolumeMounts(b *BaseComponentFields, isvc *v1beta1.InferenceService, podSpec *corev1.PodSpec, objectMeta *metav1.ObjectMeta) { + if podSpec == nil || len(podSpec.InitContainers) == 0 { + return + } + if b.BaseModel == nil || b.BaseModel.Storage == nil || b.BaseModel.Storage.Path == nil || b.BaseModelMeta == nil { + return + } + if !isvcutils.IsOriginalModelVolumeMountNecessary(objectMeta.Annotations) { + return + } + if strings.TrimSpace(isvcutils.ModelStoragePVCClaimName(isvc, b.InferenceServiceConfig)) == "" { + return + } + mountRoot := isvcutils.ModelStoragePVCMountRoot(b.InferenceServiceConfig) + storagePath := filepath.Clean(*b.BaseModel.Storage.Path) + sub := isvcutils.ModelVolumeMountSubPathForPVC(mountRoot, storagePath) + if sub == "" { + return + } + volName := b.BaseModelMeta.Name + for i := range podSpec.InitContainers { + for j := range podSpec.InitContainers[i].VolumeMounts { + vm := &podSpec.InitContainers[i].VolumeMounts[j] + if vm.Name != volName { + continue + } + if filepath.Clean(vm.MountPath) != storagePath { + continue + } + vm.SubPath = sub + } + } +} + // UpdateEnvVariables updates environment variables for the container func UpdateEnvVariables(b *BaseComponentFields, isvc *v1beta1.InferenceService, container *corev1.Container, objectMeta *metav1.ObjectMeta) { if container == nil { diff --git a/pkg/controller/v1beta1/inferenceservice/components/base_test.go b/pkg/controller/v1beta1/inferenceservice/components/base_test.go index 42360388c..75215dc49 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base_test.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base_test.go @@ -383,3 +383,59 @@ func TestUpdateVolumeMounts_FTWithInjectAddsModelEmptyDirMount(t *testing.T) { } g.Expect(names).To(gomega.ContainElement(constants.ModelEmptyDirVolumeName)) } + +func TestUpdateInitContainerBaseModelVolumeMounts_PVCSubPath(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + isvc := &v1beta1.InferenceService{ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}} + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "my-model"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + pod := &v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "wait-for-base-model", + VolumeMounts: []v1.VolumeMount{ + {Name: "my-model", MountPath: modelPath}, + }, + }, + }, + } + UpdateInitContainerBaseModelVolumeMounts(b, isvc, pod, &metav1.ObjectMeta{Annotations: map[string]string{}}) + g.Expect(pod.InitContainers[0].VolumeMounts[0].SubPath).To(gomega.Equal("my-model")) +} + +func TestUpdateInitContainerBaseModelVolumeMounts_NoPVCNoOp(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + isvc := &v1beta1.InferenceService{ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}} + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "my-model"}, + Log: logr.Discard(), + } + pod := &v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "wait", + VolumeMounts: []v1.VolumeMount{ + {Name: "my-model", MountPath: modelPath}, + }, + }, + }, + } + UpdateInitContainerBaseModelVolumeMounts(b, isvc, pod, &metav1.ObjectMeta{Annotations: map[string]string{}}) + g.Expect(pod.InitContainers[0].VolumeMounts[0].SubPath).To(gomega.BeEmpty()) +} diff --git a/pkg/controller/v1beta1/inferenceservice/components/decoder.go b/pkg/controller/v1beta1/inferenceservice/components/decoder.go index 662baad31..00d219c54 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/decoder.go +++ b/pkg/controller/v1beta1/inferenceservice/components/decoder.go @@ -301,6 +301,7 @@ func (d *Decoder) reconcilePodSpec(isvc *v1beta1.InferenceService, objectMeta *m return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&d.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecVolumes(&d.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecNodeSelector(&d.BaseComponentFields, isvc, podSpec, v1beta1.DecoderComponent) UpdateDecoderAffinity(&d.BaseComponentFields, isvc, podSpec) @@ -336,6 +337,7 @@ func (d *Decoder) reconcileWorkerPodSpec(isvc *v1beta1.InferenceService, objectM if err != nil { return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&d.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecVolumes(&d.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecNodeSelector(&d.BaseComponentFields, isvc, workerPodSpec, v1beta1.DecoderComponent) UpdateDecoderAffinity(&d.BaseComponentFields, isvc, workerPodSpec) diff --git a/pkg/controller/v1beta1/inferenceservice/components/engine.go b/pkg/controller/v1beta1/inferenceservice/components/engine.go index 71539ce5a..f87fe1ca6 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/engine.go +++ b/pkg/controller/v1beta1/inferenceservice/components/engine.go @@ -304,6 +304,7 @@ func (e *Engine) reconcilePodSpec(isvc *v1beta1.InferenceService, objectMeta *me if err != nil { return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&e.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecVolumes(&e.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecNodeSelector(&e.BaseComponentFields, isvc, podSpec, v1beta1.EngineComponent) UpdateEngineAffinity(&e.BaseComponentFields, isvc, podSpec) @@ -339,6 +340,7 @@ func (e *Engine) reconcileWorkerPodSpec(isvc *v1beta1.InferenceService, objectMe if err != nil { return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&e.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecVolumes(&e.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecNodeSelector(&e.BaseComponentFields, isvc, workerPodSpec, v1beta1.EngineComponent) UpdateEngineAffinity(&e.BaseComponentFields, isvc, workerPodSpec) From a1cadd9aee6af204ba518a9528d3ad3b3b379aa5 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Thu, 30 Apr 2026 08:51:41 -0700 Subject: [PATCH 33/35] migrate from ubuntu* to self-hosted for GHA builds --- .github/workflows/dev-images.yaml | 6 +++--- .github/workflows/openvex.yaml | 2 +- .github/workflows/pages.yml | 4 ++-- .github/workflows/pr-labeler.yml | 2 +- .github/workflows/pr-push-validation.yml | 14 +++++++------- .github/workflows/pr-validation.yml | 10 +++++----- .github/workflows/release.yaml | 10 +++++----- .github/workflows/sbom.yaml | 2 +- .github/workflows/sync-dependabot.yaml | 2 +- .github/workflows/sync-labels.yml | 2 +- .github/workflows/vsco-publish-helm-charts.yaml | 2 +- .github/workflows/vsco-publish-ome-manager.yaml | 2 +- .github/workflows/web-console.yml | 6 +++--- 13 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/dev-images.yaml b/.github/workflows/dev-images.yaml index 6b40d3617..07e8e5076 100644 --- a/.github/workflows/dev-images.yaml +++ b/.github/workflows/dev-images.yaml @@ -20,7 +20,7 @@ env: jobs: build-and-push-images: - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: component: @@ -111,7 +111,7 @@ jobs: publish-dev-charts: needs: build-and-push-images - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -178,7 +178,7 @@ jobs: notify: needs: [build-and-push-images, publish-dev-charts] - runs-on: ubuntu-latest + runs-on: self-hosted if: always() steps: - name: Summary diff --git a/.github/workflows/openvex.yaml b/.github/workflows/openvex.yaml index 88c105adf..b3d3467cd 100644 --- a/.github/workflows/openvex.yaml +++ b/.github/workflows/openvex.yaml @@ -16,7 +16,7 @@ permissions: jobs: generate-vex: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 7fad26eef..45b986235 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -23,7 +23,7 @@ concurrency: jobs: # Build job build: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v6 @@ -75,7 +75,7 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest + runs-on: self-hosted needs: build steps: - name: Deploy to GitHub Pages diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index f15254a7e..ad35064e3 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -10,7 +10,7 @@ permissions: jobs: label: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Auto-label by file changes uses: actions/labeler@v6 diff --git a/.github/workflows/pr-push-validation.yml b/.github/workflows/pr-push-validation.yml index 2ffff71ec..4519c0eb5 100644 --- a/.github/workflows/pr-push-validation.yml +++ b/.github/workflows/pr-push-validation.yml @@ -12,7 +12,7 @@ permissions: jobs: lint: name: Lint - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -45,7 +45,7 @@ jobs: test: name: Test - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: test-suite: [unit] # TODO: Add 'integration' back once fixed @@ -99,7 +99,7 @@ jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: component: [ome-manager, model-agent, multinode-prober, ome-agent] @@ -144,7 +144,7 @@ jobs: docker-build: name: Docker Build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: image: [ome-image, model-agent-image, multinode-prober-image, ome-agent-image] @@ -189,7 +189,7 @@ jobs: security-scan: name: Security Scan - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -212,7 +212,7 @@ jobs: license-check: name: License Check - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -233,7 +233,7 @@ jobs: all-checks-passed: name: All CI Checks Passed needs: [lint, test, build, docker-build, security-scan, license-check] - runs-on: ubuntu-latest + runs-on: self-hosted if: always() steps: - name: Check all job statuses diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f3e943fce..c1f887444 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -18,7 +18,7 @@ concurrency: jobs: pre-commit-checks: name: Pre-commit Checks - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 10 steps: - name: Checkout code @@ -79,7 +79,7 @@ jobs: test-and-build: name: Test and Build - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 20 # Remove dependency so tests always run independently steps: @@ -141,7 +141,7 @@ jobs: docker-validation: name: Docker Build Validation - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 15 needs: pre-commit-checks strategy: @@ -193,7 +193,7 @@ jobs: # Optional multi-arch validation - only runs when 'test-multiarch' label is present docker-multiarch-validation: name: Docker Multi-Arch Build Validation - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 30 needs: pre-commit-checks if: contains(github.event.pull_request.labels.*.name, 'test-multiarch') @@ -252,7 +252,7 @@ jobs: summary: name: PR Validation Summary - runs-on: ubuntu-latest + runs-on: self-hosted needs: [pre-commit-checks, test-and-build, docker-validation] if: always() steps: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0802e281b..410a343b8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,7 +25,7 @@ env: jobs: prepare: - runs-on: ubuntu-latest + runs-on: self-hosted outputs: version: ${{ steps.version.outputs.version }} tag: ${{ steps.version.outputs.tag }} @@ -45,7 +45,7 @@ jobs: build-images: needs: prepare - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: component: @@ -124,7 +124,7 @@ jobs: publish-helm-charts: needs: [prepare, build-images] - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -192,7 +192,7 @@ jobs: generate-sboms: needs: [prepare, build-images] - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -221,7 +221,7 @@ jobs: create-release: needs: [prepare, build-images, publish-helm-charts, generate-sboms] - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/sbom.yaml b/.github/workflows/sbom.yaml index 49667c4d0..b7e0f0c85 100644 --- a/.github/workflows/sbom.yaml +++ b/.github/workflows/sbom.yaml @@ -15,7 +15,7 @@ permissions: jobs: generate-sbom: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/sync-dependabot.yaml b/.github/workflows/sync-dependabot.yaml index dccf58f60..4d6be8d19 100644 --- a/.github/workflows/sync-dependabot.yaml +++ b/.github/workflows/sync-dependabot.yaml @@ -18,7 +18,7 @@ permissions: jobs: sync: if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 70c3aad6c..02ee82364 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -14,7 +14,7 @@ permissions: jobs: sync: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v6 diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index 786b210e7..462c29312 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -36,7 +36,7 @@ env: jobs: publish-charts: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v6 diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 5a3393873..3684d347d 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -40,7 +40,7 @@ # #jobs: # build-push-ome-manager: -# runs-on: ubuntu-latest +# runs-on: self-hosted # steps: # - name: Checkout # uses: actions/checkout@v6 diff --git a/.github/workflows/web-console.yml b/.github/workflows/web-console.yml index 8295f75b3..c14a39e02 100644 --- a/.github/workflows/web-console.yml +++ b/.github/workflows/web-console.yml @@ -24,7 +24,7 @@ jobs: # Frontend checks - lint, typecheck, build, test frontend: name: Frontend - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 15 steps: - name: Checkout code @@ -64,7 +64,7 @@ jobs: # Backend checks - build and vet backend: name: Backend - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 10 steps: - name: Checkout code @@ -97,7 +97,7 @@ jobs: # Summary job summary: name: Web Console CI Summary - runs-on: ubuntu-latest + runs-on: self-hosted needs: [frontend, backend] if: always() steps: From 0ae7b0b5ac8fbb5fd671eece7c7cad7258fb5796 Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 5 May 2026 16:54:27 -0700 Subject: [PATCH 34/35] add fine-tuned adapter for using s3 direct backend --- .../pod/fine_tuned_adapter_injector.go | 15 ++++++++-- .../pod/fine_tuned_adapter_injector_test.go | 30 +++++++++++++++++++ .../docs/reference/labels-and-annotations.md | 2 +- 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go diff --git a/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go b/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go index 941896df8..c98d1fd97 100644 --- a/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go +++ b/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go @@ -19,6 +19,14 @@ const ( fineTunedAdapterConfigMapKeyName = "fineTunedAdapter" ) +// skipFineTunedAdapterInit returns true when the OCI fine-tuned-adapter init container must not run. +// That init image only understands oci:// object storage; PVC paths, s3:// (e.g. Mountpoint CSI on the pod), +// hf://, etc. are delivered without it. +func skipFineTunedAdapterInit(storageURI string) bool { + u := strings.TrimSpace(storageURI) + return u == "" || !strings.HasPrefix(u, storage.OCIStoragePrefix) +} + // FineTunedAdapterInjector represents configuration parameters for the Fine-Tuned Adapter. type FineTunedAdapterInjector struct { Image string `json:"image" validate:"required"` @@ -48,11 +56,12 @@ func newFineTunedAdapterInjector(configMap *v1.ConfigMap, client client.Client) // InjectFineTunedAdapter injects the fine-tuned weight initialization container into the pod if necessary. func (fa *FineTunedAdapterInjector) InjectFineTunedAdapter(pod *v1.Pod) error { if fineTunedWeightName, ok := pod.ObjectMeta.Annotations[constants.FineTunedAdapterInjectionKey]; ok && len(fineTunedWeightName) > 0 { - // OCI adapter init is not used for PVC-backed weights (already on disk). Skip before validate() - // so clusters without fineTunedAdapter ConfigMap (Image/CompartmentId/…) do not deny pod admission. + // OCI adapter init only downloads oci:// objects. Skip before validate() for PVC (on-disk), + // s3:// (e.g. LoRA via Mountpoint CSI), hf://, etc., so clusters without fineTunedAdapter + // ConfigMap (Image/CompartmentId/…) do not deny pod admission. ftw, err := isvcutils.GetFineTunedWeight(fa.client, fineTunedWeightName) if err == nil && ftw.Spec.Storage != nil && ftw.Spec.Storage.StorageUri != nil { - if strings.HasPrefix(strings.TrimSpace(*ftw.Spec.Storage.StorageUri), storage.PVCStoragePrefix) { + if skipFineTunedAdapterInit(*ftw.Spec.Storage.StorageUri) { return nil } } diff --git a/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go b/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go new file mode 100644 index 000000000..75d068429 --- /dev/null +++ b/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go @@ -0,0 +1,30 @@ +package pod + +import ( + "testing" +) + +func TestSkipFineTunedAdapterInit(t *testing.T) { + t.Parallel() + cases := []struct { + name string + uri string + skip bool + }{ + {"empty", "", true}, + {"whitespace", " ", true}, + {"pvc", "pvc://harmony-ome-models/qwen3-vl-8b-instruct", true}, + {"s3", "s3://vsco-lora-adapters-dev/studio/harmony/qwen3-vl-8b-instruct/v0p2", true}, + {"hf", "hf://org/repo@main", true}, + {"oci", "oci://namespace/bucket@object", false}, + {"oci_trimmed", " oci://ns/bucket@obj ", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := skipFineTunedAdapterInit(tc.uri); got != tc.skip { + t.Fatalf("skipFineTunedAdapterInit(%q) = %v, want %v", tc.uri, got, tc.skip) + } + }) + } +} diff --git a/site/content/en/docs/reference/labels-and-annotations.md b/site/content/en/docs/reference/labels-and-annotations.md index 7aebd75b3..4639171a4 100644 --- a/site/content/en/docs/reference/labels-and-annotations.md +++ b/site/content/en/docs/reference/labels-and-annotations.md @@ -30,7 +30,7 @@ These annotations are used to configure InferenceService behavior: | Annotation | Description | |-------------------------------------------------|------------------------------------------------------| | `ome.io/inject-model-init` | Enables injection of model initialization containers | -| `ome.io/inject-fine-tuned-adapter` | Enables injection of fine-tuned adapter containers | +| `ome.io/inject-fine-tuned-adapter` | When set on engine pods, may trigger the OCI fine-tuned-adapter init container only if the referenced `FineTunedWeight` uses `oci://` storage; `pvc://`, `s3://`, `hf://`, and other non-OCI schemes are skipped (weights are already supplied elsewhere, e.g. CSI mounts). | | `ome.io/inject-serving-sidecar` | Enables injection of serving sidecar containers | | `ome.io/fine-tuned-weight-ft-strategy` | Specifies the fine-tuning strategy for weights | | `ome.io/base-model-name` | Specifies the base model name | From 6d0fdd9c30afc28fef93ef696ca4229ce146d80c Mon Sep 17 00:00:00 2001 From: Peter Ossian Date: Tue, 5 May 2026 16:57:25 -0700 Subject: [PATCH 35/35] bump to 0.1.5-vsco7 --- .github/workflows/vsco-publish-helm-charts.yaml | 12 ++++++------ .github/workflows/vsco-publish-ome-manager.yaml | 4 ++-- charts/ome-resources/values.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml index 462c29312..0e4f814f4 100644 --- a/.github/workflows/vsco-publish-helm-charts.yaml +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -4,8 +4,8 @@ # Packages ome-crd, ome-resources, ome-serving and pushes OCI charts to: # oci://ghcr.io/vsco/charts # -# Use a Helm semver for chart_version (e.g. 0.1.5-vsco6), not a leading "v" on chart_version. -# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco6" and +# Use a Helm semver for chart_version (e.g. 0.1.5-vsco7), not a leading "v" on chart_version. +# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco7" and # repository = "oci://ghcr.io/vsco/charts" # in terraform/modules/eks/backend-services/ome.tf # @@ -18,13 +18,13 @@ on: workflow_dispatch: inputs: chart_version: - description: 'Helm chart semver (e.g. 0.1.5-vsco6) — no leading v' + description: 'Helm chart semver (e.g. 0.1.5-vsco7) — no leading v' required: true - default: '0.1.5-vsco6' + default: '0.1.5-vsco7' image_tag: - description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco6)' + description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco7)' required: true - default: 'v0.1.5-vsco6' + default: 'v0.1.5-vsco7' permissions: contents: read diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml index 3684d347d..5a5f29f4f 100644 --- a/.github/workflows/vsco-publish-ome-manager.yaml +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -10,7 +10,7 @@ ## ## Usage: ## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow -## Default tag matches local manual pushes: v0.1.5-vsco6. Override "image_tag" +## Default tag matches local manual pushes: v0.1.5-vsco7. Override "image_tag" ## for a new release, or clear it to use git short SHA instead. ## ## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. @@ -26,7 +26,7 @@ # image_tag: # description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' # required: false -# default: 'v0.1.5-vsco6' +# default: 'v0.1.5-vsco7' # #permissions: # contents: read diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index 9ac9cc69d..d43b71d00 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -6,7 +6,7 @@ global: - name: ghcr-creds ome: - version: &defaultVersion v0.1.5-vsco6 + version: &defaultVersion v0.1.5-vsco7 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false"