diff --git a/internal/exec/terraform_generate_backends.go b/internal/exec/terraform_generate_backends.go index bebef51a8bd..0790f75a1ec 100644 --- a/internal/exec/terraform_generate_backends.go +++ b/internal/exec/terraform_generate_backends.go @@ -12,6 +12,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/function/literalsource" + "github.com/cloudposse/atmos/pkg/hcl" log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/schema" @@ -412,7 +413,7 @@ func ExecuteTerraformGenerateBackends( return err } } else if format == "backend-config" { - err = u.WriteToFileAsHcl(backendFileAbsolutePath, backendSection, 0o644) + err = hcl.WriteAttributesFile(backendFileAbsolutePath, backendSection, DefaultFileMode) if err != nil { return err } diff --git a/internal/exec/terraform_generate_backends_test.go b/internal/exec/terraform_generate_backends_test.go index 9b2ae42c4c5..0988095e7fa 100644 --- a/internal/exec/terraform_generate_backends_test.go +++ b/internal/exec/terraform_generate_backends_test.go @@ -12,6 +12,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" cfg "github.com/cloudposse/atmos/pkg/config" + "github.com/cloudposse/atmos/pkg/hcl" log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" @@ -532,7 +533,7 @@ func TestBackendGenerationWithNestedMaps(t *testing.T) { // Write backend-config backendFile := filepath.Join(componentDir, "backend.tfbackend") - err = u.WriteToFileAsHcl(backendFile, backendConfig, 0o644) + err = hcl.WriteAttributesFile(backendFile, backendConfig, 0o644) require.NoError(t, err) // Read and verify diff --git a/internal/exec/terraform_generate_varfiles.go b/internal/exec/terraform_generate_varfiles.go index 7f4fd837f98..52d883379cc 100644 --- a/internal/exec/terraform_generate_varfiles.go +++ b/internal/exec/terraform_generate_varfiles.go @@ -12,6 +12,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/function/literalsource" + "github.com/cloudposse/atmos/pkg/hcl" atmosio "github.com/cloudposse/atmos/pkg/io" log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/perf" @@ -351,7 +352,7 @@ func ExecuteTerraformGenerateVarfiles( return err } } else if format == "hcl" { - err = u.WriteToFileAsHcl(fileAbsolutePath, varsToWrite, filePermissions) + err = hcl.WriteAttributesFile(fileAbsolutePath, varsToWrite, filePermissions) if err != nil { return err } diff --git a/pkg/generator/writer.go b/pkg/generator/writer.go index f5f2d13a8f9..d7a4ce29547 100644 --- a/pkg/generator/writer.go +++ b/pkg/generator/writer.go @@ -7,6 +7,7 @@ import ( "path/filepath" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/hcl" "github.com/cloudposse/atmos/pkg/perf" u "github.com/cloudposse/atmos/pkg/utils" ) @@ -80,7 +81,7 @@ func (w *FileWriter) WriteHCL(dir, filename string, data map[string]any) error { defer perf.Track(nil, "generator.FileWriter.WriteHCL")() path := filepath.Join(dir, filename) - return u.WriteToFileAsHcl(path, data, w.fileMode) + return hcl.WriteAttributesFile(path, data, w.fileMode) } // MockWriter is a test implementation that captures written content. diff --git a/pkg/hcl/doc.go b/pkg/hcl/doc.go index 24a2861e040..26ee9909fb5 100644 --- a/pkg/hcl/doc.go +++ b/pkg/hcl/doc.go @@ -6,6 +6,10 @@ // edits to .tf files follow the same shape as structural edits to YAML stack // manifests. // +// It also renders Go maps as HCL attribute files (EncodeAttributes and +// WriteAttributesFile), the format Terraform reads from .tfvars and +// -backend-config files. +// // Example usage: // // out, err := hcl.SetAttribute(content, "resource.aws_instance.web.instance_type", `"t3.micro"`) diff --git a/pkg/hcl/errors.go b/pkg/hcl/errors.go index c12b80adc80..3d64ae94c71 100644 --- a/pkg/hcl/errors.go +++ b/pkg/hcl/errors.go @@ -23,4 +23,8 @@ var ( // ErrHCLWriteFile is returned when a file cannot be written. ErrHCLWriteFile = errors.New("failed to write file") + + // ErrHCLEncodeFailed is returned when a value cannot be encoded as HCL + // attributes. + ErrHCLEncodeFailed = errors.New("failed to encode value as HCL") ) diff --git a/pkg/hcl/write.go b/pkg/hcl/write.go new file mode 100644 index 00000000000..31f4d5fa625 --- /dev/null +++ b/pkg/hcl/write.go @@ -0,0 +1,73 @@ +package hcl + +import ( + "encoding/json" + "fmt" + "os" + "sort" + + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" + ctyjson "github.com/zclconf/go-cty/cty/json" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// EncodeAttributes renders the top-level keys of data as HCL attributes, the +// format Terraform expects in .tfvars and -backend-config files. Nested maps +// become object values and lists, including lists of maps, become tuples. +// Keys are written in sorted order. +func EncodeAttributes(data any) ([]byte, error) { + defer perf.Track(nil, "hcl.EncodeAttributes")() + + j, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf(errWrapFmt, ErrHCLEncodeFailed, err) + } + + ty, err := ctyjson.ImpliedType(j) + if err != nil { + return nil, fmt.Errorf(errWrapFmt, ErrHCLEncodeFailed, err) + } + if !ty.IsObjectType() { + return nil, fmt.Errorf("%w: HCL attributes can only be generated from a map, got %s", ErrHCLEncodeFailed, ty.FriendlyName()) + } + + value, err := ctyjson.Unmarshal(j, ty) + if err != nil { + return nil, fmt.Errorf(errWrapFmt, ErrHCLEncodeFailed, err) + } + + names := make([]string, 0, len(ty.AttributeTypes())) + for name := range ty.AttributeTypes() { + names = append(names, name) + } + sort.Strings(names) + + file := hclwrite.NewEmptyFile() + body := file.Body() + for _, name := range names { + if !hclsyntax.ValidIdentifier(name) { + return nil, fmt.Errorf("%w: invalid attribute name %q", ErrHCLEncodeFailed, name) + } + body.SetAttributeValue(name, value.GetAttr(name)) + } + + return hclwrite.Format(file.Bytes()), nil +} + +// WriteAttributesFile encodes data with EncodeAttributes and writes it to +// path, replacing any previous content. +func WriteAttributesFile(path string, data any, mode os.FileMode) error { + defer perf.Track(nil, "hcl.WriteAttributesFile")() + + content, err := EncodeAttributes(data) + if err != nil { + return err + } + + if err := os.WriteFile(path, content, mode); err != nil { + return fmt.Errorf(errWrapFmt, ErrHCLWriteFile, err) + } + return nil +} diff --git a/pkg/hcl/write_test.go b/pkg/hcl/write_test.go new file mode 100644 index 00000000000..224e414d3cb --- /dev/null +++ b/pkg/hcl/write_test.go @@ -0,0 +1,117 @@ +package hcl + +import ( + "os" + "path/filepath" + "testing" + + hcl2 "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty" +) + +// parseTfvars parses generated content the same way Terraform reads a .tfvars file. +func parseTfvars(t *testing.T, content []byte) map[string]cty.Value { + t.Helper() + + file, diags := hclsyntax.ParseConfig(content, "test.tfvars", hcl2.InitialPos) + require.False(t, diags.HasErrors(), "generated HCL does not parse: %s\n%s", diags.Error(), content) + + attrs, diags := file.Body.JustAttributes() + require.False(t, diags.HasErrors(), "generated HCL is not a flat list of attributes: %s\n%s", diags.Error(), content) + + values := make(map[string]cty.Value, len(attrs)) + for name, attr := range attrs { + value, diags := attr.Expr.Value(nil) + require.False(t, diags.HasErrors(), "attribute %q: %s", name, diags.Error()) + values[name] = value + } + return values +} + +func TestEncodeAttributesNestedValues(t *testing.T) { + content, err := EncodeAttributes(map[string]any{ + "enabled": true, + "mymap": map[string]any{ + "somekey": map[string]any{ + "somevalue": true, + }, + }, + "subnets": []any{ + map[string]any{"name": "a", "cidr": "10.0.0.0/24"}, + map[string]any{"name": "b", "cidr": "10.0.1.0/24"}, + }, + "tags": map[string]any{"Name": "vpc", "kubernetes.io/role/elb": "1"}, + "template": "${not_interpolated}", + "ratio": 0.123456789, + "nothing": nil, + }) + require.NoError(t, err) + values := parseTfvars(t, content) + + assert.Equal(t, cty.True, values["enabled"]) + assert.Equal(t, cty.True, values["mymap"].GetAttr("somekey").GetAttr("somevalue")) + + subnets := values["subnets"] + require.Equal(t, 2, subnets.LengthInt()) + assert.Equal(t, "a", subnets.Index(cty.NumberIntVal(0)).GetAttr("name").AsString()) + assert.Equal(t, "10.0.1.0/24", subnets.Index(cty.NumberIntVal(1)).GetAttr("cidr").AsString()) + + assert.Equal(t, "1", values["tags"].GetAttr("kubernetes.io/role/elb").AsString()) + assert.Equal(t, "${not_interpolated}", values["template"].AsString()) + assert.True(t, values["ratio"].Equals(cty.NumberFloatVal(0.123456789)).True()) + assert.True(t, values["nothing"].IsNull()) +} + +func TestEncodeAttributesSortsKeys(t *testing.T) { + content, err := EncodeAttributes(map[string]any{"b": 1, "a": "x"}) + require.NoError(t, err) + assert.Equal(t, "a = \"x\"\nb = 1\n", string(content)) +} + +func TestEncodeAttributesErrors(t *testing.T) { + tests := []struct { + name string + data any + }{ + {name: "list", data: []any{"a", "b"}}, + {name: "scalar", data: "a"}, + {name: "unsupported value", data: map[string]any{"ch": make(chan int)}}, + {name: "invalid attribute name", data: map[string]any{"bad.key": "x"}}, + {name: "attribute name with a space", data: map[string]any{"bad key": "x"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := EncodeAttributes(tt.data) + assert.ErrorIs(t, err, ErrHCLEncodeFailed) + }) + } +} + +func TestWriteAttributesFileTruncatesExistingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "test.tfvars") + + require.NoError(t, WriteAttributesFile(path, map[string]any{ + "enabled": true, + "environment": "ue2", + "description": "a long description that makes the first file noticeably longer", + }, 0o644)) + + require.NoError(t, WriteAttributesFile(path, map[string]any{"enabled": false}, 0o644)) + + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "enabled = false\n", string(got)) +} + +func TestWriteAttributesFileErrors(t *testing.T) { + dir := t.TempDir() + + err := WriteAttributesFile(filepath.Join(dir, "missing", "test.tfvars"), map[string]any{"a": 1}, 0o644) + assert.ErrorIs(t, err, ErrHCLWriteFile) + + err = WriteAttributesFile(filepath.Join(dir, "test.tfvars"), []any{1}, 0o644) + assert.ErrorIs(t, err, ErrHCLEncodeFailed) +} diff --git a/pkg/utils/hcl_utils.go b/pkg/utils/hcl_utils.go index 2eef35b6dc7..37e74261d7f 100644 --- a/pkg/utils/hcl_utils.go +++ b/pkg/utils/hcl_utils.go @@ -2,77 +2,13 @@ package utils import ( "os" - "strings" - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/printer" - jsonParser "github.com/hashicorp/hcl/json/parser" "github.com/hashicorp/hcl/v2/hclwrite" log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/perf" ) -// WriteToFileAsHcl converts the provided value to HCL (HashiCorp Language) and writes it to the specified file. -func WriteToFileAsHcl( - filePath string, - data any, - fileMode os.FileMode, -) error { - defer perf.Track(nil, "utils.WriteToFileAsHcl")() - - astree, err := ConvertToHclAst(data) - if err != nil { - return err - } - - f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode) - if err != nil { - return err - } - - defer func(f *os.File) { - err := f.Close() - if err != nil { - log.Warn(err.Error()) - } - }(f) - - err = printer.Fprint(f, astree) - if err != nil { - return err - } - - return nil -} - -// ConvertToHclAst converts the provided value to an HCL abstract syntax tree. -func ConvertToHclAst(data any) (ast.Node, error) { - defer perf.Track(nil, "utils.ConvertToHclAst")() - - j, err := ConvertToJSONFast(data) - if err != nil { - return nil, err - } - - astree, err := jsonParser.Parse([]byte(j)) - if err != nil { - return nil, err - } - - // Remove the double quotes around the terraform variable names (the double quotes come from JSON keys) - // since they will be written to the terraform varfiles and terraform does not like it. - if objectList, ok := astree.Node.(*ast.ObjectList); ok { - for _, item := range objectList.Items { - for i, key := range item.Keys { - item.Keys[i].Token.Text = strings.Replace(key.Token.Text, "\"", "", -1) - } - } - } - - return astree.Node, nil -} - // WriteTerraformBackendConfigToFileAsHcl writes the provided Terraform backend config to the specified file. // https://dev.to/pdcommunity/write-terraform-files-in-go-with-hclwrite-2e1j // https://pkg.go.dev/github.com/hashicorp/hcl/v2/hclwrite