diff --git a/pkg/component/helm/chart.go b/pkg/component/helm/chart.go index a3d000ece1f..3fffbc27779 100644 --- a/pkg/component/helm/chart.go +++ b/pkg/component/helm/chart.go @@ -54,6 +54,9 @@ type chartSpec struct { Values map[string]any // IncludeCRDs includes CRDs from the chart's crds/ directory in the output. IncludeCRDs bool + // planCRDs appends only missing install-only CRDs during a live install plan. + // It is nil for upgrades, offline renders, and actual release actions. + planCRDs func(context.Context, helmchart.Charter) (string, error) // DependencyUpdate fetches missing dependencies before loading the chart. DependencyUpdate bool // Repositories lists declarative chart repositories for "repo/name" refs. @@ -192,7 +195,15 @@ func runInstall(ctx context.Context, client *action.Install, settings *cli.EnvSe if !ok { return "", fmt.Errorf("%w: unexpected release type %T", errUtils.ErrHelmRenderFailed, rel) } - return renderReleaseManifest(rendered), nil + manifest := renderReleaseManifest(rendered) + if spec.planCRDs != nil { + crds, err := spec.planCRDs(operationCtx, loaded) + if err != nil { + return "", err + } + manifest += crds + } + return manifest, nil } // loadChart loads a chart and verifies that every declared dependency is present. diff --git a/pkg/component/helm/client.go b/pkg/component/helm/client.go index 5642346aa61..f80caa0e1ab 100644 --- a/pkg/component/helm/client.go +++ b/pkg/component/helm/client.go @@ -319,32 +319,40 @@ func resolveUpgradeChartRef(client *action.Upgrade, spec *chartSpec) string { // getDeployedManifest returns the manifest of the currently deployed release so // it can serve as the baseline for a live diff. A release that does not exist -// yet yields an empty manifest (every object is reported as added) rather than an -// error. This is the only diff path that requires cluster access. +// yet yields an empty manifest rather than an error. Release-based plans also +// use release existence to distinguish install-only CRDs from managed resources. func getDeployedManifest(releaseName, namespace string, runners ...*recovery.Runner) (string, error) { - defer perf.Track(nil, "helm.getDeployedManifest")() + deployed, err := getDeployedRelease(releaseName, namespace, runners...) + if err != nil || deployed == nil { + return "", err + } + return renderReleaseManifest(deployed), nil +} +// getDeployedRelease preserves release existence even for CRD-only charts whose +// stored manifest is empty. Only ErrReleaseNotFound denotes a new installation. +func getDeployedRelease(releaseName, namespace string, runners ...*recovery.Runner) (*release.Release, error) { + defer perf.Track(nil, "helm.getDeployedRelease")() var runner *recovery.Runner if len(runners) > 0 { runner = runners[0] } actx, err := newRecoveryActionContext(namespace, runner) if err != nil { - return "", err + return nil, err } - rel, err := action.NewGet(actx.cfg).Run(releaseName) if err != nil { if errors.Is(err, driver.ErrReleaseNotFound) { - return "", nil + return nil, nil } - return "", fmt.Errorf("failed to get deployed Helm release %q: %w", releaseName, err) + return nil, fmt.Errorf("failed to get deployed Helm release %q: %w", releaseName, err) } deployed, ok := rel.(*release.Release) if !ok { - return "", fmt.Errorf("%w: unexpected release type %T", errUtils.ErrHelmRenderFailed, rel) + return nil, fmt.Errorf("%w: unexpected release type %T", errUtils.ErrHelmRenderFailed, rel) } - return renderReleaseManifest(deployed), nil + return deployed, nil } // deleteRelease uninstalls the release. diff --git a/pkg/component/helm/executor.go b/pkg/component/helm/executor.go index bcc052fd7b7..bf69d996f03 100644 --- a/pkg/component/helm/executor.go +++ b/pkg/component/helm/executor.go @@ -415,12 +415,11 @@ func runDiff( ctx, cancel := context.WithTimeout(callerCtx, renderTimeout) defer cancel() - desired, err := renderChartManifest(ctx, spec) + baseline, renderSpec, err := prepareDiff(ctx, atmosConfig, info, flags, spec) if err != nil { return "", err } - - baseline, err := resolveDiffBaseline(callerCtx, atmosConfig, info, flags, spec) + desired, err := renderChartManifest(ctx, renderSpec) if err != nil { return "", err } diff --git a/pkg/component/helm/plan_crds.go b/pkg/component/helm/plan_crds.go new file mode 100644 index 00000000000..ab27b272cc0 --- /dev/null +++ b/pkg/component/helm/plan_crds.go @@ -0,0 +1,175 @@ +package helm + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + + helmchart "helm.sh/helm/v4/pkg/chart" + chartv2 "helm.sh/helm/v4/pkg/chart/v2" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + kubeschema "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/dynamic" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// prepareDiff keeps offline comparisons complete, but makes release plans match +// Helm's lifecycle: crds/ files are installed only when a release is absent and +// only when the CRD itself does not already exist. Templates remain untouched. +func prepareDiff(ctx context.Context, config *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo, + flags map[string]any, spec *chartSpec, +) (string, *chartSpec, error) { + renderSpec := *spec + against := flagString(flags, flagAgainst) + if flagString(flags, flagFromManifest) != "" || (against != "" && against != againstRelease) { + baseline, err := resolveDiffBaseline(ctx, config, info, flags, spec) + return baseline, &renderSpec, err + } + if err := ctx.Err(); err != nil { + return "", nil, err + } + deployed, err := getDeployedRelease(spec.ReleaseName, spec.Namespace, spec.recovery) + if err != nil { + return "", nil, err + } + renderSpec.IncludeCRDs = false + if deployed != nil { + return renderReleaseManifest(deployed), &renderSpec, nil + } + if spec.IncludeCRDs { + lookup := newCRDExistenceLookup(spec) + renderSpec.planCRDs = func(ctx context.Context, chart helmchart.Charter) (string, error) { + return missingCRDManifests(ctx, chart, lookup) + } + } + return "", &renderSpec, nil +} + +// newCRDExistenceLookup initializes its client lazily so charts without crds/ +// require no CRD API access. GET avoids requiring cluster-wide list permission. +func newCRDExistenceLookup(spec *chartSpec) func(context.Context, string) (bool, error) { + var client dynamic.Interface + return func(ctx context.Context, name string) (bool, error) { + if client == nil { + settings := newSettingsForNamespace(spec.Namespace) + configureRecoveryTransport(settings, spec.recovery) + restConfig, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return false, err + } + client, err = dynamic.NewForConfig(restConfig) + if err != nil { + return false, err + } + } + _, err := client.Resource(kubeschema.GroupVersionResource{ + Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions", + }).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + return err == nil, err + } +} + +// missingCRDManifests uses Helm's processed chart (including enabled subcharts), +// not a kind-based filter that could also suppress managed template resources. +func missingCRDManifests(ctx context.Context, chart helmchart.Charter, + exists func(context.Context, string) (bool, error), +) (string, error) { + loaded, ok := chart.(*chartv2.Chart) + if !ok { + return "", fmt.Errorf("%w: cannot plan CRDs for chart type %T", errUtils.ErrHelmDiffFailed, chart) + } + var out bytes.Buffer + for _, crd := range loaded.CRDObjects() { + content, err := missingCRDFile(ctx, crd, exists) + if err != nil { + return "", err + } + out.WriteString(content) + } + return out.String(), nil +} + +const crdDecoderBufferSize = 4096 + +func missingCRDFile(ctx context.Context, crd chartv2.CRD, + exists func(context.Context, string) (bool, error), +) (string, error) { + var out bytes.Buffer + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(crd.File.Data), crdDecoderBufferSize) + for { + if err := ctx.Err(); err != nil { + return "", err + } + var object unstructured.Unstructured + if err := decoder.Decode(&object); errors.Is(err, io.EOF) { + break + } else if err != nil { + return "", fmt.Errorf("decode CRD file %q: %w", crd.Filename, err) + } + content, err := missingCRDDocument(ctx, &object, crd.Filename, exists) + if err != nil { + return "", fmt.Errorf("plan CRD file %q: %w", crd.Filename, err) + } + out.WriteString(content) + } + return out.String(), nil +} + +// missingCRDDocument expands Lists just as Helm's Kubernetes resource builder +// does, including nested Lists, before checking individual CRDs. +func missingCRDDocument(ctx context.Context, object *unstructured.Unstructured, source string, + exists func(context.Context, string) (bool, error), +) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + if object.IsList() { + list, err := object.ToList() + if err != nil { + return "", err + } + var out bytes.Buffer + for i := range list.Items { + content, err := missingCRDDocument(ctx, &list.Items[i], source, exists) + if err != nil { + return "", err + } + out.WriteString(content) + } + return out.String(), nil + } + content, err := missingCRDObject(ctx, object, exists) + if err != nil || len(content) == 0 { + return "", err + } + return fmt.Sprintf("---\n# Source: %s\n%s\n", source, content), nil +} + +func missingCRDObject(ctx context.Context, object *unstructured.Unstructured, + exists func(context.Context, string) (bool, error), +) ([]byte, error) { + if len(object.Object) == 0 { + return nil, nil + } + if object.GetKind() != "CustomResourceDefinition" || object.GetName() == "" { + return nil, fmt.Errorf("%w: expected a named CustomResourceDefinition", errUtils.ErrHelmDiffFailed) + } + found, err := exists(ctx, object.GetName()) + if err != nil { + return nil, fmt.Errorf("check existing CRD %q: %w", object.GetName(), err) + } + if found { + return nil, nil + } + return object.MarshalJSON() +} diff --git a/pkg/component/helm/plan_crds_test.go b/pkg/component/helm/plan_crds_test.go new file mode 100644 index 00000000000..76ac4e5d605 --- /dev/null +++ b/pkg/component/helm/plan_crds_test.go @@ -0,0 +1,229 @@ +package helm + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + helmchart "helm.sh/helm/v4/pkg/chart" + "helm.sh/helm/v4/pkg/chart/common" + chartv2 "helm.sh/helm/v4/pkg/chart/v2" + rcommon "helm.sh/helm/v4/pkg/release/common" + release "helm.sh/helm/v4/pkg/release/v1" + + "github.com/cloudposse/atmos/pkg/schema" +) + +const planTestCRD = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.example.com +spec: + group: example.com + scope: Namespaced + names: + kind: Widget + plural: widgets + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +` + +func crdPlanSpec(t *testing.T, managed bool) *chartSpec { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "crds"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "templates"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "Chart.yaml"), []byte("apiVersion: v2\nname: demo\nversion: 0.1.0\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "crds", "widgets.yaml"), []byte(planTestCRD), 0o600)) + if managed { + require.NoError(t, os.WriteFile(filepath.Join(dir, "templates", "managed.yaml"), []byte(strings.ReplaceAll(planTestCRD, "widgets", "managedwidgets")), 0o600)) + } + return &chartSpec{Chart: dir, ReleaseName: "demo", Namespace: "default", IncludeCRDs: true} +} + +func TestCRDPlanExistingRelease(t *testing.T) { + for _, managed := range []bool{false, true} { + name := "CRD-only release" + if managed { + name = "managed template CRD" + } + t.Run(name, func(t *testing.T) { + spec := crdPlanSpec(t, managed) + storedSpec := *spec + storedSpec.IncludeCRDs = false + baseline, err := renderManifest(context.Background(), &storedSpec) + require.NoError(t, err) + actx := memoryActionContext(t) + stubActionContext(t, actx) + require.NoError(t, actx.cfg.Releases.Create(&release.Release{Name: "demo", Namespace: "default", Version: 1, Manifest: baseline, Info: &release.Info{Status: rcommon.StatusDeployed}})) + old, prepared, err := prepareDiff(context.Background(), &schema.AtmosConfiguration{}, &schema.ConfigAndStacksInfo{}, nil, spec) + require.NoError(t, err) + require.False(t, prepared.IncludeCRDs) + require.Nil(t, prepared.planCRDs) + require.True(t, spec.IncludeCRDs, "do not mutate the shared spec") + desired, err := renderManifest(context.Background(), prepared) + require.NoError(t, err) + _, changed, err := unifiedDiff(old, desired, "default", 3) + require.NoError(t, err) + assert.False(t, changed) + if managed { + require.NoError(t, os.WriteFile(filepath.Join(spec.Chart, "templates", "managed.yaml"), []byte(strings.ReplaceAll(strings.ReplaceAll(planTestCRD, "widgets", "managedwidgets"), "scope: Namespaced", "scope: Cluster")), 0o600)) + desired, err = renderManifest(context.Background(), prepared) + require.NoError(t, err) + diff, changed, err := unifiedDiff(old, desired, "default", 3) + require.NoError(t, err) + assert.True(t, changed) + assert.Contains(t, diff, "scope: Cluster") + } + }) + } +} + +func TestCRDPlanNewReleaseAndOffline(t *testing.T) { + spec := crdPlanSpec(t, false) + stubActionContext(t, memoryActionContext(t)) + _, prepared, err := prepareDiff(context.Background(), &schema.AtmosConfiguration{}, &schema.ConfigAndStacksInfo{}, nil, spec) + require.NoError(t, err) + require.NotNil(t, prepared.planCRDs) + require.False(t, prepared.IncludeCRDs) + for _, found := range []bool{false, true} { + prepared.planCRDs = func(ctx context.Context, chart helmchart.Charter) (string, error) { + return missingCRDManifests(ctx, chart, func(context.Context, string) (bool, error) { return found, nil }) + } + rendered, err := renderManifest(context.Background(), prepared) + require.NoError(t, err) + assert.Equal(t, !found, strings.Contains(rendered, "widgets.example.com")) + } + baselineFile := filepath.Join(t.TempDir(), "baseline.yaml") + require.NoError(t, os.WriteFile(baselineFile, []byte(""), 0o600)) + _, prepared, err = prepareDiff(context.Background(), &schema.AtmosConfiguration{}, &schema.ConfigAndStacksInfo{}, map[string]any{flagFromManifest: baselineFile, flagAgainst: againstRelease}, spec) + require.NoError(t, err) + require.True(t, prepared.IncludeCRDs) + require.Nil(t, prepared.planCRDs) + rendered, err := renderManifest(context.Background(), prepared) + require.NoError(t, err) + assert.Contains(t, rendered, "widgets.example.com") +} + +func TestMissingCRDManifests(t *testing.T) { + chart := &chartv2.Chart{Metadata: &chartv2.Metadata{Name: "parent"}} + child := &chartv2.Chart{Metadata: &chartv2.Metadata{Name: "child"}, Files: []*common.File{{Name: "crds/widgets.yaml", Data: []byte(planTestCRD + "\n---\n" + strings.ReplaceAll(planTestCRD, "widgets", "gadgets"))}}} + chart.AddDependency(child) + var names []string + rendered, err := missingCRDManifests(context.Background(), chart, func(_ context.Context, name string) (bool, error) { + names = append(names, name) + return name == "widgets.example.com", nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"widgets.example.com", "gadgets.example.com"}, names) + assert.NotContains(t, rendered, "widgets.example.com") + assert.Contains(t, rendered, "gadgets.example.com") + denied := errors.New("forbidden") + _, err = missingCRDManifests(context.Background(), chart, func(context.Context, string) (bool, error) { return false, denied }) + require.ErrorIs(t, err, denied) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = missingCRDManifests(ctx, chart, func(context.Context, string) (bool, error) { t.Fatal("unexpected lookup"); return false, nil }) + require.ErrorIs(t, err, context.Canceled) +} + +func TestCRDExistenceLookup(t *testing.T) { + for _, code := range []int{http.StatusOK, http.StatusNotFound, http.StatusForbidden, http.StatusInternalServerError} { + t.Run(fmt.Sprint(code), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/widgets.example.com", r.URL.Path) + assert.Equal(t, http.MethodGet, r.Method) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + if code == http.StatusOK { + _, _ = fmt.Fprint(w, `{"apiVersion":"apiextensions.k8s.io/v1","kind":"CustomResourceDefinition","metadata":{"name":"widgets.example.com"}}`) + } else { + _, _ = fmt.Fprintf(w, `{"apiVersion":"v1","kind":"Status","status":"Failure","code":%d}`, code) + } + })) + defer server.Close() + path := filepath.Join(t.TempDir(), "kubeconfig") + require.NoError(t, os.WriteFile(path, []byte(fmt.Sprintf(`apiVersion: v1 +kind: Config +clusters: +- name: test + cluster: + server: %s +contexts: +- name: test + context: + cluster: test +current-context: test +`, server.URL)), 0o600)) + t.Setenv("KUBECONFIG", path) + lookup := newCRDExistenceLookup(&chartSpec{Namespace: "default"}) + found, err := lookup(context.Background(), "widgets.example.com") + if code == http.StatusOK || code == http.StatusNotFound { + require.NoError(t, err) + assert.Equal(t, code == http.StatusOK, found) + } else { + require.Error(t, err) + assert.False(t, found) + } + }) + } +} + +func TestCRDPlanRespectsSubchartCondition(t *testing.T) { + spec := crdPlanSpec(t, false) + child := filepath.Join(spec.Chart, "charts", "child") + require.NoError(t, os.MkdirAll(filepath.Join(child, "crds"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(child, "Chart.yaml"), []byte("apiVersion: v2\nname: child\nversion: 0.1.0\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(child, "crds", "widgets.yaml"), []byte(strings.ReplaceAll(planTestCRD, "widgets", "childwidgets")), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(spec.Chart, "Chart.yaml"), []byte("apiVersion: v2\nname: demo\nversion: 0.1.0\ndependencies:\n- name: child\n version: 0.1.0\n condition: child.enabled\n"), 0o600)) + stubActionContext(t, memoryActionContext(t)) + for _, enabled := range []bool{false, true} { + spec.Values = map[string]any{"child": map[string]any{"enabled": enabled}} + _, prepared, err := prepareDiff(context.Background(), &schema.AtmosConfiguration{}, &schema.ConfigAndStacksInfo{}, nil, spec) + require.NoError(t, err) + prepared.planCRDs = func(ctx context.Context, chart helmchart.Charter) (string, error) { + return missingCRDManifests(ctx, chart, func(context.Context, string) (bool, error) { return false, nil }) + } + rendered, err := renderManifest(context.Background(), prepared) + require.NoError(t, err) + assert.Equal(t, enabled, strings.Contains(rendered, "childwidgets.example.com")) + } +} + +func TestMissingCRDList(t *testing.T) { + for _, nested := range []bool{false, true} { + t.Run(fmt.Sprintf("nested=%t", nested), func(t *testing.T) { + items := "apiVersion: v1\nkind: List\nitems:\n" + for _, name := range []string{"widgets", "gadgets"} { + items += "- " + strings.ReplaceAll(strings.TrimSpace(strings.ReplaceAll(planTestCRD, "widgets", name)), "\n", "\n ") + "\n" + } + if nested { + items = "apiVersion: v1\nkind: List\nitems:\n- " + strings.ReplaceAll(strings.TrimSpace(items), "\n", "\n ") + "\n" + } + chart := &chartv2.Chart{Metadata: &chartv2.Metadata{Name: "lists"}, Files: []*common.File{{Name: "crds/list.yaml", Data: []byte(items)}}} + var names []string + rendered, err := missingCRDManifests(context.Background(), chart, func(_ context.Context, name string) (bool, error) { + names = append(names, name) + return name == "widgets.example.com", nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"widgets.example.com", "gadgets.example.com"}, names) + assert.NotContains(t, rendered, "widgets.example.com") + assert.Contains(t, rendered, "gadgets.example.com") + assert.NotContains(t, rendered, `"kind":"List"`) + }) + } +} diff --git a/website/docs/cli/commands/helm/helm-diff.mdx b/website/docs/cli/commands/helm/helm-diff.mdx index 1f2848d64dd..56661a58584 100644 --- a/website/docs/cli/commands/helm/helm-diff.mdx +++ b/website/docs/cli/commands/helm/helm-diff.mdx @@ -45,7 +45,7 @@ precedence (`--from-manifest` → `--against` → deployed release):
Deployed release (default)
-
The manifest of the currently installed release, read from the cluster. A release that does not exist yet shows every object as added. This is the only mode that needs cluster access.
+
The manifest of the currently installed release, read from the cluster. A release that does not exist yet shows its template resources and any missing install-only CRDs as added. This is the only mode that needs cluster access.
`--from-manifest=`
A local baseline manifest file. Fully offline — no cluster, no git.
@@ -54,6 +54,21 @@ precedence (`--from-manifest` → `--against` → deployed release):
The manifests currently published in a non-cluster provision target (e.g. a git deployment repository). Reads the target's current state and diffs against it — the producer side of a GitOps workflow. Offline (git access only). Without :<name> the provision.default target is used.
+## Custom resource definitions + +Release-based `diff` and `plan` follow Helm's handling of the chart's `crds/` +directory, including enabled subcharts: + +- For an existing release, these install-only CRDs are omitted from the diff. + Helm does not upgrade them or store them in the release manifest. +- For a new release, Atmos checks each CRD by name and shows only those missing + from the cluster. This requires permission to get the chart's CRDs; lookup + errors fail the plan rather than treating the CRDs as missing. + +CRDs declared in `templates/` remain part of the normal diff because Helm manages +those resources. `helm template`, local-file comparisons, and GitOps target +comparisons continue to include the chart's full CRD manifests. + ## Flags