From 772bd6af0099601e7e52b901998c441b1174f19c Mon Sep 17 00:00:00 2001 From: Marek Aufart Date: Tue, 25 Aug 2026 15:21:27 +0200 Subject: [PATCH] Support kustomize fragments in crane transform Extending crane transform so users can attach an inline kustomize fragment to individual stages. The fragment should be merged into the kustomization.yaml that crane generates for that stage, and it should be configurable both via a repeatable CLI flag and via the transform instructions file, following the existing per-stage optionals pattern. Fixes: https://github.com/migtools/crane/issues/879 Signed-off-by: Marek Aufart --- cmd/transform/kustomize_fragment_test.go | 110 +++++++++ cmd/transform/transform.go | 86 +++++-- docs/kustomize-fragments.md | 124 ++++++++++ internal/kustomize/merge.go | 134 ++++++++++ internal/kustomize/merge_test.go | 231 ++++++++++++++++++ internal/transform/instructions.go | 22 +- internal/transform/kustomize_fragment_test.go | 210 ++++++++++++++++ internal/transform/orchestrator.go | 62 ++++- internal/transform/writer.go | 14 ++ 9 files changed, 958 insertions(+), 35 deletions(-) create mode 100644 cmd/transform/kustomize_fragment_test.go create mode 100644 docs/kustomize-fragments.md create mode 100644 internal/kustomize/merge.go create mode 100644 internal/kustomize/merge_test.go create mode 100644 internal/transform/kustomize_fragment_test.go diff --git a/cmd/transform/kustomize_fragment_test.go b/cmd/transform/kustomize_fragment_test.go new file mode 100644 index 00000000..250149b3 --- /dev/null +++ b/cmd/transform/kustomize_fragment_test.go @@ -0,0 +1,110 @@ +package transform + +import ( + "strings" + "testing" +) + +func TestParseStageKustomize(t *testing.T) { + tests := []struct { + name string + values []string + wantErr string + check func(t *testing.T, result map[string]map[string]interface{}) + }{ + { + name: "single stage json", + values: []string{`KubernetesPlugin={"namespace": "dest-ns", "commonLabels": {"app": "crane"}}`}, + check: func(t *testing.T, result map[string]map[string]interface{}) { + frag := result["KubernetesPlugin"] + if frag["namespace"] != "dest-ns" { + t.Errorf("namespace: got %v", frag["namespace"]) + } + }, + }, + { + name: "single stage yaml", + values: []string{"CustomEdits=namespace: dest-ns\ncommonLabels:\n app: crane\n"}, + check: func(t *testing.T, result map[string]map[string]interface{}) { + frag := result["CustomEdits"] + if frag["namespace"] != "dest-ns" { + t.Errorf("namespace: got %v", frag["namespace"]) + } + }, + }, + { + name: "multiple stages", + values: []string{ + `KubernetesPlugin={"namespace": "a"}`, + `RegistryPlugin={"namespace": "b"}`, + }, + check: func(t *testing.T, result map[string]map[string]interface{}) { + if len(result) != 2 { + t.Errorf("expected 2 stages, got %d", len(result)) + } + }, + }, + { + name: "missing equals sign", + values: []string{"KubernetesPlugin"}, + wantErr: "expected format StageName=YAML", + }, + { + name: "empty stage name", + values: []string{`={"namespace": "x"}`}, + wantErr: "stage name is empty", + }, + { + name: "empty fragment", + values: []string{`KubernetesPlugin=`}, + wantErr: "empty", + }, + { + name: "list fragment rejected", + values: []string{`KubernetesPlugin=["a", "b"]`}, + wantErr: "must be a mapping", + }, + { + name: "duplicate stage", + values: []string{ + `KubernetesPlugin={"namespace": "a"}`, + `KubernetesPlugin={"namespace": "b"}`, + }, + wantErr: "duplicate", + }, + { + name: "fragment with commas is not split", + values: []string{`KubernetesPlugin={"images": [{"name": "a", "newName": "b"}], "namespace": "ns"}`}, + check: func(t *testing.T, result map[string]map[string]interface{}) { + frag := result["KubernetesPlugin"] + if frag["namespace"] != "ns" { + t.Errorf("namespace: got %v", frag["namespace"]) + } + if _, ok := frag["images"].([]interface{}); !ok { + t.Errorf("images: expected list, got %T", frag["images"]) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := parseStageKustomize(tt.values) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.check != nil { + tt.check(t, result) + } + }) + } +} diff --git a/cmd/transform/transform.go b/cmd/transform/transform.go index 7be947b1..066eb15b 100644 --- a/cmd/transform/transform.go +++ b/cmd/transform/transform.go @@ -41,13 +41,14 @@ type Options struct { } type Flags struct { - ExportDir string `mapstructure:"export-dir"` - PluginDir string `mapstructure:"plugin-dir"` - TransformDir string `mapstructure:"transform-dir"` - SkipPlugins []string `mapstructure:"skip-plugins"` - OptionalFlags string `mapstructure:"optional-flags"` - StageOptionals []string `mapstructure:"stage-optionals"` - Overwrite bool `mapstructure:"overwrite"` + ExportDir string `mapstructure:"export-dir"` + PluginDir string `mapstructure:"plugin-dir"` + TransformDir string `mapstructure:"transform-dir"` + SkipPlugins []string `mapstructure:"skip-plugins"` + OptionalFlags string `mapstructure:"optional-flags"` + StageOptionals []string `mapstructure:"stage-optionals"` + StageKustomize []string `mapstructure:"stage-kustomize"` + Overwrite bool `mapstructure:"overwrite"` // Kustomize arguments KustomizeArgs string `mapstructure:"kustomize-args"` // Instructions file @@ -170,6 +171,7 @@ func addFlagsForOptions(o *Flags, cmd *cobra.Command) { cmd.Flags().StringVar(&o.OptionalFlags, "optional-flags", "", "JSON string holding flag value pairs to be passed to all plugins (e.g. '{\"registry-replacement\": \"docker.io=quay.io\"}')") cmd.Flags().StringArrayVar(&o.StageOptionals, "stage-optionals", nil, "Per-stage optional flags as StageName=JSON, repeatable (e.g. --stage-optionals 'KubernetesPlugin={\"registry-replacement\":\"docker.io=quay.io\"}')") + cmd.Flags().StringArrayVar(&o.StageKustomize, "stage-kustomize", nil, "Per-stage inline kustomize fragment as StageName=YAML|JSON, repeatable. The fragment is merged into the stage's generated kustomization.yaml (resources/patches are appended, other fields override). E.g. --stage-kustomize 'KubernetesPlugin={\"namespace\":\"dest-ns\",\"commonLabels\":{\"app\":\"crane\"}}'") // Kustomize arguments cmd.Flags().StringVar(&o.KustomizeArgs, "kustomize-args", "", "Additional arguments for kustomize (e.g., '--enable-helm --helm-command=helm3')") @@ -203,15 +205,19 @@ func (o *Options) run() error { } if o.InstructionsFile != "" && len(o.RequestedStages) > 0 { - log.Debugf("Cannot use --instructions-file together with positional stage arguments") + log.Debugf("Cannot use --instructions-file together with positional stage arguments") return fmt.Errorf("use either --instructions-file or positional stage arguments, not both") } if o.InstructionsFile != "" && len(o.StageOptionals) > 0 { return fmt.Errorf("use either --instructions-file or --stage-optionals, not both") } + if o.InstructionsFile != "" && len(o.StageKustomize) > 0 { + return fmt.Errorf("use either --instructions-file or --stage-kustomize, not both") + } var instructionStages []string var instructionStageOptionals map[string]map[string]string + var instructionStageKustomize map[string]map[string]interface{} if o.InstructionsFile != "" { instructionsFilePath, err := filepath.Abs(o.InstructionsFile) if err != nil { @@ -228,6 +234,7 @@ func (o *Options) run() error { if err != nil { return fmt.Errorf("invalid instructions file %q: %w", instructionsFilePath, err) } + instructionStageKustomize = cfg.StageKustomize() } // Parse optional flags var optionalFlags map[string]string @@ -257,6 +264,20 @@ func (o *Options) run() error { stageOptionalFlags = instructionStageOptionals } + // Parse per-stage kustomize fragments from CLI + var stageKustomizeFragments map[string]map[string]interface{} + if len(o.StageKustomize) > 0 { + stageKustomizeFragments, err = parseStageKustomize(o.StageKustomize) + if err != nil { + return err + } + } + + // Use instruction file per-stage kustomize fragments if present, otherwise CLI + if instructionStageKustomize != nil { + stageKustomizeFragments = instructionStageKustomize + } + // Parse and validate kustomize arguments kustomizeArgs, err := kustomize.ParseAndValidateArgs(o.KustomizeArgs) if err != nil { @@ -266,17 +287,18 @@ func (o *Options) run() error { // Create orchestrator orchestrator := &internalTransform.Orchestrator{ - Log: log.WithField("command", "transform").Logger, - ExportDir: exportDir, - TransformDir: transformDir, - PluginDir: pluginDir, - SkipPlugins: o.SkipPlugins, - OptionalFlags: optionalFlags, - StageOptionalFlags: stageOptionalFlags, - Overwrite: o.Overwrite, - CraneVersion: "v1.0.0", // TODO: Get from build version - NewlyCreatedStages: make(map[string]bool), - KustomizeArgs: kustomizeArgs, + Log: log.WithField("command", "transform").Logger, + ExportDir: exportDir, + TransformDir: transformDir, + PluginDir: pluginDir, + SkipPlugins: o.SkipPlugins, + OptionalFlags: optionalFlags, + StageOptionalFlags: stageOptionalFlags, + Overwrite: o.Overwrite, + CraneVersion: "v1.0.0", // TODO: Get from build version + NewlyCreatedStages: make(map[string]bool), + KustomizeArgs: kustomizeArgs, + StageKustomizeFragments: stageKustomizeFragments, } // Determine which stages to run @@ -418,6 +440,32 @@ func parseStageOptionals(values []string) (map[string]map[string]string, error) return result, nil } +// parseStageKustomize parses --stage-kustomize values from "StageName=YAML|JSON" +// format into a map of stage name to inline kustomize fragment. +func parseStageKustomize(values []string) (map[string]map[string]interface{}, error) { + result := make(map[string]map[string]interface{}, len(values)) + for _, v := range values { + stageName, fragStr, found := strings.Cut(v, "=") + if !found { + return nil, fmt.Errorf("invalid --stage-kustomize value %q: expected format StageName=YAML", v) + } + + if stageName == "" { + return nil, fmt.Errorf("invalid --stage-kustomize value %q: stage name is empty", v) + } + if _, exists := result[stageName]; exists { + return nil, fmt.Errorf("duplicate --stage-kustomize for stage %q", stageName) + } + + fragment, err := kustomize.ParseFragment(fragStr) + if err != nil { + return nil, fmt.Errorf("invalid --stage-kustomize for stage %q: %w", stageName, err) + } + result[stageName] = fragment + } + return result, nil +} + // Returns an extras map with lowercased keys, since any keys coming from the config file // are lower-cased by viper func optionalFlagsToLowerChecked(inFlags map[string]string) (map[string]string, error) { diff --git a/docs/kustomize-fragments.md b/docs/kustomize-fragments.md new file mode 100644 index 00000000..c7f5bc75 --- /dev/null +++ b/docs/kustomize-fragments.md @@ -0,0 +1,124 @@ +# Per-stage kustomize fragments + +`crane transform` can merge an inline **kustomize fragment** into the +`kustomization.yaml` that is generated for a stage. This lets you inject extra +kustomize configuration (namespace, common labels/annotations, images, name +prefixes, additional resources/patches, …) without hand-editing the generated +files after every run. + +The fragment is provided inline — either through a CLI flag or in the +transform instructions file — following the same per-stage pattern as +[`--stage-optionals`](./multistage-pipeline.md). + +## How it works + +For each stage, crane generates a `kustomization.yaml` containing `resources` +and `patches`. When a fragment is configured for that stage, it is merged into +the generated file using these rules: + +| Field | Merge behaviour | +|-------|-----------------| +| `resources` | Fragment entries are **appended** to the generated ones (de-duplicated by value). | +| `patches` | Fragment entries are **appended** to the generated ones. | +| `apiVersion`, `kind` | Kept from the generated file; fragment values are ignored. | +| any other field | Fragment value **replaces** the generated value. | + +Stages without a fragment are left byte-for-byte unchanged. + +The stage is identified by its **plugin/base name** (e.g. `KubernetesPlugin`, +`CustomEdits`) — the same key used by `--stage-optionals` — not by the numbered +directory name (`10_KubernetesPlugin`). + +## CLI flag + +``` +--stage-kustomize 'StageName=' +``` + +The flag is **repeatable** (once per stage). The value after `=` is a kustomize +fragment as a mapping. JSON is valid YAML, so either form works; JSON is usually +easier to pass on a single command line. + +### Example: namespace + common labels + +```sh +crane transform KubernetesPlugin \ + --stage-kustomize 'KubernetesPlugin={"namespace":"dest-ns","commonLabels":{"app":"crane"}}' +``` + +Resulting `transform/10_KubernetesPlugin/kustomization.yaml`: + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app: crane +namespace: dest-ns +resources: +- input/ConfigMap__v1_default_demo.yaml +``` + +### Example: image overrides (multi-line YAML value) + +```sh +crane transform KubernetesPlugin \ + --stage-kustomize 'KubernetesPlugin= +images: +- name: nginx + newName: quay.io/mirror/nginx + newTag: "1.27" +' +``` + +### Example: multiple stages + +```sh +crane transform \ + --stage-kustomize 'KubernetesPlugin={"namespace":"dest-ns"}' \ + --stage-kustomize 'CustomEdits={"commonAnnotations":{"origin":"crane"}}' +``` + +## Instructions file + +Add a `kustomize:` block to a stage entry, alongside the existing `optionals:` +field: + +```yaml +# instructions.yaml +stages: + - name: KubernetesPlugin + optionals: + registry-replacement: "docker.io=quay.io" + kustomize: + namespace: dest-ns + commonLabels: + app: crane + - name: CustomEdits + kustomize: + commonAnnotations: + origin: instructions +``` + +Run it with: + +```sh +crane transform --instructions-file instructions.yaml +``` + +> `--instructions-file` cannot be combined with `--stage-kustomize` (or +> `--stage-optionals`). When an instructions file is used, its `kustomize:` +> blocks take precedence. + +## Validation & errors + +- The fragment must be a **mapping** — a list or scalar is rejected. +- A fragment referencing a stage that is not part of the run fails with + `per-stage kustomize fragment references unknown stage "..."`. +- `resources`/`patches` in a fragment must be lists. +- Extra `resources` must point to files that exist relative to the stage + directory, otherwise the subsequent `kustomize build` fails. + +## See also + +- [Multistage pipeline](./multistage-pipeline.md) — stages, `--stage-optionals`, + and the instructions file format. diff --git a/internal/kustomize/merge.go b/internal/kustomize/merge.go new file mode 100644 index 00000000..528469b3 --- /dev/null +++ b/internal/kustomize/merge.go @@ -0,0 +1,134 @@ +package kustomize + +import ( + "fmt" + "strings" + + "sigs.k8s.io/yaml" +) + +// protectedFields are kustomization keys that must never be overridden by a +// user-provided fragment, since changing them would break the stage pipeline. +var protectedFields = map[string]bool{ + "apiVersion": true, + "kind": true, +} + +// listMergeFields are keys whose fragment values are appended to the generated +// values instead of replacing them. +var listMergeFields = map[string]bool{ + "resources": true, + "patches": true, +} + +// ParseFragment parses an inline kustomize fragment (YAML or JSON, since JSON is +// a subset of YAML) into a generic map. It rejects empty input and any fragment +// whose root is not a mapping (e.g. a list or a scalar). +func ParseFragment(raw string) (map[string]interface{}, error) { + if strings.TrimSpace(raw) == "" { + return nil, fmt.Errorf("kustomize fragment is empty") + } + var probe interface{} + if err := yaml.Unmarshal([]byte(raw), &probe); err != nil { + return nil, fmt.Errorf("invalid kustomize fragment: %w", err) + } + out, ok := probe.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("kustomize fragment must be a mapping") + } + return out, nil +} + +// MergeFragment merges a user-provided kustomize fragment into the generated +// kustomization.yaml bytes and returns the merged YAML. +// +// Merge rules: +// - "resources" and "patches": fragment entries are appended to the generated +// entries. Resources are de-duplicated by value. +// - "apiVersion" and "kind": kept from the generated base, fragment values are +// ignored. +// - any other key: the fragment value replaces the generated value. +// +// An empty fragment returns the base unchanged. +func MergeFragment(base []byte, fragment map[string]interface{}) ([]byte, error) { + if len(fragment) == 0 { + return base, nil + } + + merged := map[string]interface{}{} + if err := yaml.Unmarshal(base, &merged); err != nil { + return nil, fmt.Errorf("failed to parse generated kustomization.yaml: %w", err) + } + + for key, val := range fragment { + if protectedFields[key] { + continue + } + if listMergeFields[key] { + mergedList, err := appendList(key, merged[key], val) + if err != nil { + return nil, err + } + merged[key] = mergedList + continue + } + merged[key] = val + } + + out, err := yaml.Marshal(merged) + if err != nil { + return nil, fmt.Errorf("failed to marshal merged kustomization.yaml: %w", err) + } + return out, nil +} + +// appendList appends the fragment list to the base list. For "resources" the +// result is de-duplicated by string value, preserving base-then-fragment order. +func appendList(key string, base, fragment interface{}) (interface{}, error) { + baseList, err := toList(key, base) + if err != nil { + return nil, err + } + fragList, err := toList(key, fragment) + if err != nil { + return nil, err + } + + result := make([]interface{}, 0, len(baseList)+len(fragList)) + result = append(result, baseList...) + + if key == "resources" { + seen := make(map[string]bool, len(baseList)) + for _, item := range baseList { + if s, ok := item.(string); ok { + seen[s] = true + } + } + for _, item := range fragList { + if s, ok := item.(string); ok { + if seen[s] { + continue + } + seen[s] = true + } + result = append(result, item) + } + return result, nil + } + + result = append(result, fragList...) + return result, nil +} + +// toList coerces a value into a list, treating nil as an empty list and +// rejecting non-list values with a descriptive error. +func toList(key string, v interface{}) ([]interface{}, error) { + if v == nil { + return nil, nil + } + list, ok := v.([]interface{}) + if !ok { + return nil, fmt.Errorf("kustomize fragment field %q must be a list", key) + } + return list, nil +} diff --git a/internal/kustomize/merge_test.go b/internal/kustomize/merge_test.go new file mode 100644 index 00000000..aac8bb6e --- /dev/null +++ b/internal/kustomize/merge_test.go @@ -0,0 +1,231 @@ +package kustomize + +import ( + "strings" + "testing" + + "sigs.k8s.io/yaml" +) + +const baseKustomization = `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- input/a.yaml +- input/b.yaml +patches: +- path: patches/p1.yaml + target: + kind: Deployment + name: web +` + +// unmarshalYAML is a small helper to turn merged bytes back into a generic map. +func unmarshalYAML(t *testing.T, data []byte) map[string]interface{} { + t.Helper() + var out map[string]interface{} + if err := yaml.Unmarshal(data, &out); err != nil { + t.Fatalf("failed to unmarshal merged YAML: %v\n%s", err, data) + } + return out +} + +func TestParseFragment(t *testing.T) { + tests := []struct { + name string + raw string + wantErr string + check func(t *testing.T, m map[string]interface{}) + }{ + { + name: "json object", + raw: `{"namespace": "dest-ns", "commonLabels": {"app": "crane"}}`, + check: func(t *testing.T, m map[string]interface{}) { + if m["namespace"] != "dest-ns" { + t.Errorf("namespace: got %v", m["namespace"]) + } + }, + }, + { + name: "yaml object", + raw: "namespace: dest-ns\ncommonLabels:\n app: crane\n", + check: func(t *testing.T, m map[string]interface{}) { + if m["namespace"] != "dest-ns" { + t.Errorf("namespace: got %v", m["namespace"]) + } + }, + }, + { + name: "empty", + raw: " ", + wantErr: "empty", + }, + { + name: "list root", + raw: `["a", "b"]`, + wantErr: "must be a mapping", + }, + { + name: "scalar root", + raw: `just-a-string`, + wantErr: "must be a mapping", + }, + { + name: "invalid yaml", + raw: "namespace: : :\n - broken", + wantErr: "invalid kustomize fragment", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, err := ParseFragment(tt.raw) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.check != nil { + tt.check(t, m) + } + }) + } +} + +func TestMergeFragment_AddsScalarAndMapFields(t *testing.T) { + fragment := map[string]interface{}{ + "namespace": "dest-ns", + "commonLabels": map[string]interface{}{"app": "crane"}, + "namePrefix": "pre-", + } + + out, err := MergeFragment([]byte(baseKustomization), fragment) + if err != nil { + t.Fatalf("MergeFragment failed: %v", err) + } + m := unmarshalYAML(t, out) + + if m["namespace"] != "dest-ns" { + t.Errorf("namespace: got %v", m["namespace"]) + } + if m["namePrefix"] != "pre-" { + t.Errorf("namePrefix: got %v", m["namePrefix"]) + } + labels, ok := m["commonLabels"].(map[string]interface{}) + if !ok || labels["app"] != "crane" { + t.Errorf("commonLabels: got %v", m["commonLabels"]) + } + // Base fields preserved. + if m["kind"] != "Kustomization" { + t.Errorf("kind changed: got %v", m["kind"]) + } +} + +func TestMergeFragment_AppendsResources(t *testing.T) { + fragment := map[string]interface{}{ + "resources": []interface{}{"input/b.yaml", "extra/c.yaml"}, + } + + out, err := MergeFragment([]byte(baseKustomization), fragment) + if err != nil { + t.Fatalf("MergeFragment failed: %v", err) + } + m := unmarshalYAML(t, out) + + resources, ok := m["resources"].([]interface{}) + if !ok { + t.Fatalf("resources not a list: %T", m["resources"]) + } + // input/b.yaml is de-duplicated; extra/c.yaml appended. + want := []string{"input/a.yaml", "input/b.yaml", "extra/c.yaml"} + if len(resources) != len(want) { + t.Fatalf("expected %d resources, got %d: %v", len(want), len(resources), resources) + } + for i, w := range want { + if resources[i] != w { + t.Errorf("resources[%d]: expected %q, got %v", i, w, resources[i]) + } + } +} + +func TestMergeFragment_AppendsPatches(t *testing.T) { + fragment := map[string]interface{}{ + "patches": []interface{}{ + map[string]interface{}{ + "path": "patches/p2.yaml", + "target": map[string]interface{}{ + "kind": "Service", + "name": "web", + }, + }, + }, + } + + out, err := MergeFragment([]byte(baseKustomization), fragment) + if err != nil { + t.Fatalf("MergeFragment failed: %v", err) + } + m := unmarshalYAML(t, out) + + patches, ok := m["patches"].([]interface{}) + if !ok { + t.Fatalf("patches not a list: %T", m["patches"]) + } + if len(patches) != 2 { + t.Fatalf("expected 2 patches, got %d: %v", len(patches), patches) + } +} + +func TestMergeFragment_IgnoresProtectedFields(t *testing.T) { + fragment := map[string]interface{}{ + "apiVersion": "evil/v1", + "kind": "NotKustomization", + "namespace": "dest-ns", + } + + out, err := MergeFragment([]byte(baseKustomization), fragment) + if err != nil { + t.Fatalf("MergeFragment failed: %v", err) + } + m := unmarshalYAML(t, out) + + if m["apiVersion"] != "kustomize.config.k8s.io/v1beta1" { + t.Errorf("apiVersion overridden: got %v", m["apiVersion"]) + } + if m["kind"] != "Kustomization" { + t.Errorf("kind overridden: got %v", m["kind"]) + } + if m["namespace"] != "dest-ns" { + t.Errorf("namespace not applied: got %v", m["namespace"]) + } +} + +func TestMergeFragment_EmptyFragmentReturnsBase(t *testing.T) { + out, err := MergeFragment([]byte(baseKustomization), nil) + if err != nil { + t.Fatalf("MergeFragment failed: %v", err) + } + if string(out) != baseKustomization { + t.Errorf("expected base unchanged, got:\n%s", out) + } +} + +func TestMergeFragment_RejectsNonListResources(t *testing.T) { + fragment := map[string]interface{}{ + "resources": "input/c.yaml", // scalar, not a list + } + + _, err := MergeFragment([]byte(baseKustomization), fragment) + if err == nil { + t.Fatalf("expected error for non-list resources, got nil") + } + if !strings.Contains(err.Error(), "must be a list") { + t.Fatalf("expected 'must be a list' error, got %v", err) + } +} diff --git a/internal/transform/instructions.go b/internal/transform/instructions.go index f75b6702..b25892e7 100644 --- a/internal/transform/instructions.go +++ b/internal/transform/instructions.go @@ -26,8 +26,9 @@ var rootSequenceInstructionsRegex = regexp.MustCompile(`line ([0-9]+): cannot un // It can be specified as either a plain string (just the name) or an object // with name and optional per-stage flags. type StageEntry struct { - Name string `yaml:"name"` - Optionals map[string]string `yaml:"optionals,omitempty"` + Name string `yaml:"name"` + Optionals map[string]string `yaml:"optionals,omitempty"` + Kustomize map[string]interface{} `yaml:"kustomize,omitempty"` } type InstructionsFile struct { @@ -70,8 +71,8 @@ func (f *InstructionsFile) UnmarshalYAML(value *yamlv3.Node) error { // Check for unknown keys in the stage entry for j := 0; j+1 < len(node.Content); j += 2 { key := node.Content[j].Value - if key != "name" && key != "optionals" { - return fmt.Errorf("stage at index %d: unknown field %q (supported fields: name, optionals)", i, key) + if key != "name" && key != "optionals" && key != "kustomize" { + return fmt.Errorf("stage at index %d: unknown field %q (supported fields: name, optionals, kustomize)", i, key) } } f.Stages = append(f.Stages, entry) @@ -192,6 +193,19 @@ func (f *InstructionsFile) StageOptionals() (map[string]map[string]string, error return result, nil } +// StageKustomize returns a map of stage name to inline kustomize fragment for +// stages that have a per-stage kustomize fragment defined. Stages without a +// fragment are omitted. +func (f *InstructionsFile) StageKustomize() map[string]map[string]interface{} { + result := make(map[string]map[string]interface{}) + for _, s := range f.Stages { + if len(s.Kustomize) > 0 { + result[s.Name] = s.Kustomize + } + } + return result +} + // GenerateStageDirNames converts ordered stage tokens into deterministic stage // directory names using 10-step numeric prefixes (10_, 20_, 30_, ...). func GenerateStageDirNames(stageTokens []string) []string { diff --git a/internal/transform/kustomize_fragment_test.go b/internal/transform/kustomize_fragment_test.go new file mode 100644 index 00000000..70d31fd4 --- /dev/null +++ b/internal/transform/kustomize_fragment_test.go @@ -0,0 +1,210 @@ +package transform + +import ( + "os" + "path/filepath" + "strings" + "testing" + + cranelib "github.com/konveyor/crane-lib/transform" + "github.com/konveyor/crane/internal/file" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Instructions file: per-stage kustomize fragment is parsed and exposed via StageKustomize. +func TestLoadInstructions_KustomizeFragment(t *testing.T) { + tmpDir := t.TempDir() + instructionsFilePath := filepath.Join(tmpDir, "kustomize-instructions.yaml") + + content := []byte(`stages: + - name: KubernetesPlugin + kustomize: + namespace: dest-ns + commonLabels: + app: crane + - name: CustomEdits +`) + if err := os.WriteFile(instructionsFilePath, content, 0o600); err != nil { + t.Fatalf("failed to write test config: %v", err) + } + + cfg, err := LoadInstructions(instructionsFilePath) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + frags := cfg.StageKustomize() + if len(frags) != 1 { + t.Fatalf("expected 1 stage with kustomize, got %d", len(frags)) + } + frag := frags["KubernetesPlugin"] + if frag["namespace"] != "dest-ns" { + t.Errorf("namespace: got %v", frag["namespace"]) + } + if _, ok := frag["commonLabels"].(map[string]interface{}); !ok { + t.Errorf("commonLabels: expected map, got %T", frag["commonLabels"]) + } +} + +// Unknown-field error message mentions the kustomize field as supported. +func TestLoadInstructions_UnknownStageFieldMentionsKustomize(t *testing.T) { + tmpDir := t.TempDir() + instructionsFilePath := filepath.Join(tmpDir, "bad.yaml") + + content := []byte(`stages: + - name: KubernetesPlugin + bogus: value +`) + if err := os.WriteFile(instructionsFilePath, content, 0o600); err != nil { + t.Fatalf("failed to write test config: %v", err) + } + + _, err := LoadInstructions(instructionsFilePath) + if err == nil { + t.Fatalf("expected error, got nil") + } + if !strings.Contains(err.Error(), "kustomize") { + t.Fatalf("expected error to mention kustomize, got %v", err) + } +} + +func TestValidateStageKustomizeFragments(t *testing.T) { + stages := []Stage{ + {PluginName: "KubernetesPlugin", DirName: "10_KubernetesPlugin"}, + {PluginName: "CustomEdits", DirName: "20_CustomEdits"}, + } + + t.Run("known stage passes", func(t *testing.T) { + o := &Orchestrator{ + StageKustomizeFragments: map[string]map[string]interface{}{ + "KubernetesPlugin": {"namespace": "ns"}, + }, + } + if err := o.validateStageKustomizeFragments(stages); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("unknown stage fails", func(t *testing.T) { + o := &Orchestrator{ + StageKustomizeFragments: map[string]map[string]interface{}{ + "NopePlugin": {"namespace": "ns"}, + }, + } + err := o.validateStageKustomizeFragments(stages) + if err == nil { + t.Fatalf("expected error, got nil") + } + if !strings.Contains(err.Error(), "NopePlugin") { + t.Fatalf("expected error to mention NopePlugin, got %v", err) + } + }) + + t.Run("empty fragments pass", func(t *testing.T) { + o := &Orchestrator{} + if err := o.validateStageKustomizeFragments(stages); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +// Writer integration: with a fragment, the generated kustomization.yaml contains +// the merged fields; without a fragment it stays free of them. +func TestWriteStage_KustomizeFragmentMerged(t *testing.T) { + tmpDir := t.TempDir() + transformDir := filepath.Join(tmpDir, "transform") + stageName := "10_test" + + resource := unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-cm", + "namespace": "default", + }, + "data": map[string]interface{}{"k": "v"}, + }, + } + + artifact := StageArtifact{TransformArtifact: cranelib.TransformArtifact{ + Resource: resource, + HaveWhiteOut: false, + Target: cranelib.DeriveTargetFromResource(resource), + PluginName: "test-plugin", + }} + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + opts := file.PathOpts{TransformDir: transformDir} + + writer := NewKustomizeWriter(opts, stageName, logger) + writer.kustomizeFragment = map[string]interface{}{ + "namespace": "dest-ns", + "commonLabels": map[string]interface{}{"app": "crane"}, + "resources": []interface{}{"extra/manual.yaml"}, + } + + if err := writer.WriteStage([]StageArtifact{artifact}, true); err != nil { + t.Fatalf("WriteStage failed: %v", err) + } + + kustomizationPath := filepath.Join(transformDir, stageName, "kustomization.yaml") + data, err := os.ReadFile(kustomizationPath) + if err != nil { + t.Fatalf("failed to read kustomization.yaml: %v", err) + } + content := string(data) + + for _, want := range []string{"namespace: dest-ns", "commonLabels", "app: crane", "extra/manual.yaml"} { + if !strings.Contains(content, want) { + t.Errorf("kustomization.yaml missing %q:\n%s", want, content) + } + } + // apiVersion/kind must remain the generated kustomize ones. + if !strings.Contains(content, "kind: Kustomization") { + t.Errorf("kustomization.yaml lost kind: Kustomization:\n%s", content) + } +} + +// Regression: without a fragment the writer output must not contain fragment-only fields. +func TestWriteStage_NoFragmentUnchanged(t *testing.T) { + tmpDir := t.TempDir() + transformDir := filepath.Join(tmpDir, "transform") + stageName := "10_test" + + resource := unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-cm", + "namespace": "default", + }, + }, + } + + artifact := StageArtifact{TransformArtifact: cranelib.TransformArtifact{ + Resource: resource, + Target: cranelib.DeriveTargetFromResource(resource), + PluginName: "test-plugin", + }} + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + opts := file.PathOpts{TransformDir: transformDir} + + writer := NewKustomizeWriter(opts, stageName, logger) + if err := writer.WriteStage([]StageArtifact{artifact}, true); err != nil { + t.Fatalf("WriteStage failed: %v", err) + } + + data, err := os.ReadFile(filepath.Join(transformDir, stageName, "kustomization.yaml")) + if err != nil { + t.Fatalf("failed to read kustomization.yaml: %v", err) + } + if strings.Contains(string(data), "commonLabels") { + t.Errorf("unexpected fragment field in no-fragment output:\n%s", data) + } +} diff --git a/internal/transform/orchestrator.go b/internal/transform/orchestrator.go index 1a42188b..df1808e2 100644 --- a/internal/transform/orchestrator.go +++ b/internal/transform/orchestrator.go @@ -29,20 +29,24 @@ type StageArtifact struct { // Orchestrator coordinates multi-stage transform execution type Orchestrator struct { - Log *logrus.Logger - ExportDir string - TransformDir string - PluginDir string - SkipPlugins []string - OptionalFlags map[string]string + Log *logrus.Logger + ExportDir string + TransformDir string + PluginDir string + SkipPlugins []string + OptionalFlags map[string]string StageOptionalFlags map[string]map[string]string - Overwrite bool - CraneVersion string + Overwrite bool + CraneVersion string // NewlyCreatedStages tracks stages created in this run that can be overwritten // This prevents double-write errors when creating a stage and then running it NewlyCreatedStages map[string]bool // KustomizeArgs holds additional arguments for embedded kustomize (e.g. helm options) KustomizeArgs []string + // StageKustomizeFragments holds per-stage inline kustomize fragments, keyed by + // stage plugin/base name (e.g. "KubernetesPlugin"). Each fragment is merged + // into the generated kustomization.yaml for that stage. + StageKustomizeFragments map[string]map[string]interface{} } func (o *Orchestrator) validateStageOptionalFlags(stages []Stage) error { @@ -65,6 +69,35 @@ func (o *Orchestrator) validateStageOptionalFlags(stages []Stage) error { return nil } +func (o *Orchestrator) validateStageKustomizeFragments(stages []Stage) error { + if len(o.StageKustomizeFragments) == 0 { + return nil + } + known := make(map[string]bool, len(stages)) + for _, s := range stages { + known[s.PluginName] = true + } + for name := range o.StageKustomizeFragments { + if !known[name] { + names := make([]string, len(stages)) + for i, s := range stages { + names[i] = s.PluginName + } + return fmt.Errorf("per-stage kustomize fragment references unknown stage %q (known stages: %s)", name, strings.Join(names, ", ")) + } + } + return nil +} + +// resolveKustomizeFragment returns the inline kustomize fragment configured for +// the given stage, or nil if none is defined. +func (o *Orchestrator) resolveKustomizeFragment(stage Stage) map[string]interface{} { + if o.StageKustomizeFragments == nil { + return nil + } + return o.StageKustomizeFragments[stage.PluginName] +} + func (o *Orchestrator) resolveOptionalFlags(stage Stage) map[string]string { if o.StageOptionalFlags == nil { return o.OptionalFlags @@ -113,6 +146,10 @@ func (o *Orchestrator) RunMultiStage(stageSelector StageSelector) error { return err } + if err := o.validateStageKustomizeFragments(selectedStages); err != nil { + return err + } + opts := file.PathOpts{ TransformDir: o.TransformDir, ExportDir: o.ExportDir, @@ -231,6 +268,7 @@ func (o *Orchestrator) executeStage(stage Stage, inputResources []unstructured.U } writer := NewKustomizeWriter(opts, stage.DirName, o.Log) + writer.kustomizeFragment = o.resolveKustomizeFragment(stage) if err := writer.WriteStage(artifacts, forceWrite); err != nil { return err } @@ -389,10 +427,10 @@ func (o *Orchestrator) applyStageTransforms(stageDir string) ([]unstructured.Uns // Check if kustomize produced any output // Empty output means the stage has no resources to transform if len(output) == 0 { - return nil, fmt.Errorf("stage produced no resources to transform.\n"+ - "This can happen when:\n"+ - " - The stage received no input resources\n"+ - " - The plugin processed resources but produced no transformations\n"+ + return nil, fmt.Errorf("stage produced no resources to transform.\n" + + "This can happen when:\n" + + " - The stage received no input resources\n" + + " - The plugin processed resources but produced no transformations\n" + " - A previous stage filtered out all resources") } diff --git a/internal/transform/writer.go b/internal/transform/writer.go index d8bd369d..1e403ad8 100644 --- a/internal/transform/writer.go +++ b/internal/transform/writer.go @@ -11,6 +11,7 @@ import ( jsonpatch "github.com/evanphx/json-patch" "github.com/konveyor/crane-lib/transform/kustomize" "github.com/konveyor/crane/internal/file" + internalkustomize "github.com/konveyor/crane/internal/kustomize" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" @@ -21,6 +22,9 @@ type KustomizeWriter struct { opts file.PathOpts stageName string log *logrus.Logger + // kustomizeFragment is an optional inline kustomize fragment merged into the + // generated kustomization.yaml for this stage. Nil means no fragment. + kustomizeFragment map[string]interface{} } // NewKustomizeWriter creates a new KustomizeWriter for a specific stage @@ -375,6 +379,16 @@ func (w *KustomizeWriter) generateKustomizationWithComments(resources []string, return nil, err } + // Merge an optional user-provided kustomize fragment into the generated + // kustomization.yaml. When no fragment is configured the base is unchanged, + // preserving the exact output (and golden manifests) for the common case. + if len(w.kustomizeFragment) > 0 { + baseYAML, err = internalkustomize.MergeFragment(baseYAML, w.kustomizeFragment) + if err != nil { + return nil, fmt.Errorf("failed to merge kustomize fragment for stage %s: %w", w.stageName, err) + } + } + // If no whiteout comments, return as-is if len(whiteoutComments) == 0 { return baseYAML, nil