From 79e13050039b28edbdc594c76b07d1548e5edd3d Mon Sep 17 00:00:00 2001 From: Jaroslav Brazda Date: Sun, 13 Sep 2026 15:11:57 -0400 Subject: [PATCH 1/5] refactor(package): deepen selective export into internal/packaging Move the selective-export pipeline out of package create's RunE closure (manifest selection, closure/parent-container/CDI-ref inclusion, metadata-graph building, file filtering, CSV/metadata regeneration) into a new internal/packaging module with a single entry point, BuildSelectivePackage. cmd/package.go's RunE shrinks from ~370 lines of inline orchestration to a thin call into the new module; exportMetadata/exportedObject become aliases to their internal/packaging equivalents so the unrelated `package dependencies` subcommand keeps compiling unchanged. No flag, output, error-text, or file-format changes; no CI/docs impact. Adds table-driven tests for the new module in internal/packaging. --- cmd/package.go | 459 ++-------------------------- internal/packaging/build.go | 399 ++++++++++++++++++++++++ internal/packaging/build_test.go | 309 +++++++++++++++++++ internal/packaging/metadata.go | 51 ++++ internal/packaging/metadata_test.go | 89 ++++++ internal/packaging/types.go | 94 ++++++ internal/packaging/types_test.go | 74 +++++ 7 files changed, 1048 insertions(+), 427 deletions(-) create mode 100644 internal/packaging/build.go create mode 100644 internal/packaging/build_test.go create mode 100644 internal/packaging/metadata.go create mode 100644 internal/packaging/metadata_test.go create mode 100644 internal/packaging/types.go create mode 100644 internal/packaging/types_test.go diff --git a/cmd/package.go b/cmd/package.go index c583003..a43550a 100644 --- a/cmd/package.go +++ b/cmd/package.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/csv" "encoding/json" "errors" "fmt" @@ -24,6 +23,7 @@ import ( "github.com/jbrazda/iics-cli/internal/config" "github.com/jbrazda/iics-cli/internal/dependencies" "github.com/jbrazda/iics-cli/internal/output" + "github.com/jbrazda/iics-cli/internal/packaging" "github.com/jbrazda/iics-cli/internal/release" "github.com/spf13/cobra" ) @@ -403,200 +403,33 @@ func newPackageCreateCmd() *cobra.Command { var reportSelectedCount int var reportExcludedCount int if hasSelectionManifest { - if len(manifestEntries) == 0 { - return fmt.Errorf("selection manifest is empty") - } - meta, metaErr := readExportMetadata("", absSource) - if metaErr != nil { - return fmt.Errorf("reading source metadata for selective packaging: %w", metaErr) - } - - exported := make([]dependencies.ExportedObjectRef, 0, len(meta.ExportedObjects)) - for _, o := range meta.ExportedObjects { - exported = append(exported, dependencies.ExportedObjectRef{ - ObjectGUID: o.ObjectGUID, - ObjectName: o.ObjectName, - ObjectType: o.ObjectType, - Path: o.Path, - }) - } - selectedIDs, warnings, selErr := dependencies.SelectExportedObjects(manifestEntries, exported) - if selErr != nil { - return selErr - } - parentAdded := dependencies.IncludeParentContainers(exported, selectedIDs) - closureNodes := make([]dependencies.RefClosureNode, 0, len(meta.ExportedObjects)) - for _, o := range meta.ExportedObjects { - if o.ObjectGUID == "" { - continue - } - closureNodes = append(closureNodes, dependencies.RefClosureNode{ - ID: o.ObjectGUID, - Refs: o.objectRefs(), - }) - } - excludedSelectedIDs := make(map[string]bool) - if excludeFoundTransitive && len(manifestStats.ExcludedEntries) > 0 { - resolvedExcluded, _, excludedSelErr := dependencies.SelectExportedObjects(manifestStats.ExcludedEntries, exported) - if excludedSelErr != nil { - return fmt.Errorf("resolving excluded transitive-found entries: %w", excludedSelErr) - } - excludedSelectedIDs = resolvedExcluded - } - - closureAdded := 0 - closureSuppressedExcluded := 0 - if !excludeFoundTransitive { - closureAdded = dependencies.IncludeReferencedClosure(closureNodes, selectedIDs) - } else { - closureAddedIDs := dependencies.AddedIDsAfterClosure(closureNodes, selectedIDs) - closureSuppressedExcluded = len(closureAddedIDs) - if len(excludedSelectedIDs) > 0 && len(closureAddedIDs) > 0 { - // Keep a floor count based on all closure-suppressed additions and - // use excluded-entry overlap to avoid undercounting when manifest rows - // and closure additions are both present. - overlap := dependencies.CountSetIntersection(closureAddedIDs, excludedSelectedIDs) - if overlap > closureSuppressedExcluded { - closureSuppressedExcluded = overlap - } - } - } - if len(selectedIDs) == 0 { - return fmt.Errorf("no assets matched selection manifest") + buildResult, buildErr := packaging.BuildSelectivePackage( + fileContents, + absSource, + manifestEntries, + manifestStats, + packaging.BuildOptions{ + ExcludeFoundTransitive: excludeFoundTransitive, + PackageName: packageName, + IncludeTags: includeTags, + Target: target, + }, + ) + if buildErr != nil { + return buildErr } - for _, w := range warnings { + for _, w := range buildResult.Warnings { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", w) } - if verbose && excludeFoundTransitive { - totalExcluded := manifestStats.ExcludedTransitiveFound + closureSuppressedExcluded - _, _ = fmt.Fprintf( - cmd.OutOrStdout(), - "Selection filter: excluded %d transitive found rows using %s (manifest rows: %d, closure-suppressed: %d)\n", - totalExcluded, - manifestStats.SelectedStatusColumnName, - manifestStats.ExcludedTransitiveFound, - closureSuppressedExcluded, - ) - } - if verbose && parentAdded > 0 { - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Selection refinement: included %d inferred parent Project/Folder objects\n", parentAdded) - } - if verbose && closureAdded > 0 { - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Selection refinement: included %d in-package referenced dependencies\n", closureAdded) - } - cdiRefAdded := 0 - if excludeFoundTransitive { - refNodes := make([]dependencies.CDIObjectRefNode, 0, len(meta.ExportedObjects)) - for _, o := range meta.ExportedObjects { - if o.ObjectGUID == "" { - continue - } - refNodes = append(refNodes, dependencies.CDIObjectRefNode{ - ID: o.ObjectGUID, - Type: o.ObjectType, - ObjectRefs: o.objectRefs(), - }) - } - cdiRefAdded = dependencies.IncludeCDISysRefsFromObjectRefs(refNodes, selectedIDs) - if verbose && cdiRefAdded > 0 { - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Selection refinement: included %d CDI Connection/AgentGroup refs from objectRefs\n", cdiRefAdded) + if verbose { + for _, n := range buildResult.Notices { + _, _ = fmt.Fprint(cmd.OutOrStdout(), n) } } - - selectedObjects := make([]exportedObject, 0, len(selectedIDs)) - for _, o := range meta.ExportedObjects { - if selectedIDs[o.ObjectGUID] { - selectedObjects = append(selectedObjects, o) - } - } - if len(selectedObjects) == 0 { - return fmt.Errorf("selection manifest resolved no exported objects") - } - reportIncluded = exportedObjectsToManifestLog(selectedObjects) - reportSelectedCount = len(selectedObjects) - reportExcludedCount = manifestStats.ExcludedTransitiveFound + closureSuppressedExcluded - metadataObjects := selectedObjects - if !excludeFoundTransitive { - selectedNodes := make([]dependencies.ObjectRefsNode, len(selectedObjects)) - for i, o := range selectedObjects { - selectedNodes[i] = dependencies.ObjectRefsNode{ - ID: o.ObjectGUID, - ObjectRefs: o.objectRefs(), - } - } - prunedRefsByID, prunedCount := dependencies.PruneDanglingObjectRefs(selectedNodes) - for i := range selectedObjects { - if setErr := selectedObjects[i].setObjectRefs(prunedRefsByID[selectedObjects[i].ObjectGUID]); setErr != nil { - return setErr - } - } - postPruneNodes := make([]dependencies.ObjectRefsNode, len(selectedObjects)) - for i, o := range selectedObjects { - postPruneNodes[i] = dependencies.ObjectRefsNode{ - ID: o.ObjectGUID, - ObjectRefs: o.objectRefs(), - } - } - if dangling := dependencies.CountDanglingObjectRefs(postPruneNodes); dangling > 0 { - return fmt.Errorf("selection produced %d unresolved objectRefs after pruning; cannot create import-safe package", dangling) - } - if verbose && prunedCount > 0 { - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Selection refinement: pruned %d dangling metadata objectRefs\n", prunedCount) - } - } else { - metadataIDs := buildMetadataGraphForSelection(meta.ExportedObjects, selectedIDs, excludedSelectedIDs) - metadataObjectList := make([]exportedObject, 0, len(metadataIDs)) - for _, o := range meta.ExportedObjects { - if metadataIDs[o.ObjectGUID] { - metadataObjectList = append(metadataObjectList, o) - } - } - if len(metadataObjectList) == 0 { - return fmt.Errorf("selection metadata graph resolved no exported objects") - } - metadataNodes := make([]dependencies.ObjectRefsNode, len(metadataObjectList)) - for i, o := range metadataObjectList { - metadataNodes[i] = dependencies.ObjectRefsNode{ - ID: o.ObjectGUID, - ObjectRefs: o.objectRefs(), - } - } - prunedRefsByID, _ := dependencies.PruneDanglingObjectRefs(metadataNodes) - for i := range metadataObjectList { - if setErr := metadataObjectList[i].setObjectRefs(prunedRefsByID[metadataObjectList[i].ObjectGUID]); setErr != nil { - return setErr - } - } - metadataObjects = metadataObjectList - } - - filtered := filterPackageFilesForSelection(fileContents, meta.ExportedObjects, selectedIDs) - if len(filtered) == 0 { - return fmt.Errorf("no package files remained after selection filtering") - } - - finalPackageName := packageName - if finalPackageName == "" { - finalPackageName = strings.TrimSuffix(filepath.Base(target), filepath.Ext(target)) - } - - meta.Name = finalPackageName - if !includeTags { - meta.Tags = nil - } - meta.ExportedObjects = metadataObjects - metaData, marshalErr := json.MarshalIndent(meta, "", " ") - if marshalErr != nil { - return fmt.Errorf("serializing filtered exportMetadata.v2.json: %w", marshalErr) - } - filtered["exportMetadata.v2.json"] = metaData - - contentsCSV, csvErr := buildContentsOfExportPackageCSV(selectedObjects) - if csvErr != nil { - return csvErr - } - filtered["ContentsofExportPackage_"+finalPackageName+".csv"] = contentsCSV - fileContents = filtered + reportIncluded = buildResult.ReportIncluded + reportSelectedCount = buildResult.ReportSelected + reportExcludedCount = buildResult.ReportExcluded + fileContents = buildResult.Files } // 4. Generate checksum from all collected files @@ -798,134 +631,6 @@ func readPackageSelectionManifest( return entries, parseStats, true, nil } -func filterPackageFilesForSelection( - fileContents map[string][]byte, - allObjects []exportedObject, - selectedIDs map[string]bool, -) map[string][]byte { - allObjectFiles := make(map[string]bool) - selectedObjectFiles := make(map[string]bool) - for _, o := range allObjects { - candidates := dependencies.ObjectChecksumCandidates(o.Path, o.ObjectName, o.ObjectType) - for _, c := range candidates { - if _, ok := fileContents[c]; ok { - allObjectFiles[c] = true - if selectedIDs[o.ObjectGUID] { - selectedObjectFiles[c] = true - } - } - } - } - - out := make(map[string][]byte) - for path, data := range fileContents { - if path == "exportPackage.chksum" || path == "exportMetadata.v2.json" { - continue - } - if strings.HasPrefix(path, "ContentsofExportPackage_") && strings.HasSuffix(path, ".csv") { - continue - } - if allObjectFiles[path] { - if selectedObjectFiles[path] { - out[path] = data - } - continue - } - out[path] = data - } - return out -} - -func buildMetadataGraphForSelection( - allObjects []exportedObject, - selectedIDs map[string]bool, - excludedIDs map[string]bool, -) map[string]bool { - metadataIDs := make(map[string]bool, len(selectedIDs)) - for id := range selectedIDs { - metadataIDs[id] = true - } - byID := make(map[string]exportedObject, len(allObjects)) - exportedRefs := make([]dependencies.ExportedObjectRef, 0, len(allObjects)) - for _, o := range allObjects { - if o.ObjectGUID == "" { - continue - } - byID[o.ObjectGUID] = o - exportedRefs = append(exportedRefs, dependencies.ExportedObjectRef{ - ObjectGUID: o.ObjectGUID, - ObjectName: o.ObjectName, - ObjectType: o.ObjectType, - Path: o.Path, - }) - } - - queue := make([]string, 0, len(metadataIDs)) - for id := range metadataIDs { - queue = append(queue, id) - } - for len(queue) > 0 { - id := queue[0] - queue = queue[1:] - obj, ok := byID[id] - if !ok { - continue - } - for _, refID := range obj.objectRefs() { - if excludedIDs[refID] && !selectedIDs[refID] { - continue - } - if _, exists := byID[refID]; !exists { - continue - } - if metadataIDs[refID] { - continue - } - metadataIDs[refID] = true - queue = append(queue, refID) - } - } - - // Include container hierarchy needed for selected/closure objects. - _ = dependencies.IncludeParentContainers(exportedRefs, metadataIDs) - for id := range excludedIDs { - if selectedIDs[id] { - continue - } - delete(metadataIDs, id) - } - return metadataIDs -} - -func buildContentsOfExportPackageCSV(objects []exportedObject) ([]byte, error) { - rows := make([][]string, 0, len(objects)+1) - rows = append(rows, []string{"objectPath", "objectName", "objectType", "id"}) - - sorted := make([]exportedObject, len(objects)) - copy(sorted, objects) - sort.SliceStable(sorted, func(i, j int) bool { - a := client.NormalizeLocationPath(sorted[i].Path) + "/" + sorted[i].ObjectName + "." + sorted[i].ObjectType - b := client.NormalizeLocationPath(sorted[j].Path) + "/" + sorted[j].ObjectName + "." + sorted[j].ObjectType - return a < b - }) - - for _, o := range sorted { - rows = append(rows, []string{ - o.Path, - o.ObjectName, - o.ObjectType, - o.ObjectGUID, - }) - } - - var b bytes.Buffer - w := csv.NewWriter(&b) - if err := w.WriteAll(rows); err != nil { - return nil, fmt.Errorf("writing ContentsofExportPackage csv: %w", err) - } - return b.Bytes(), nil -} - // --------------------------------------------------------------------------- // package dependencies // --------------------------------------------------------------------------- @@ -950,79 +655,13 @@ var typePriority = map[string]int{ "TASKFLOW": 5, } -// exportMetadata represents the exportMetadata.v2.json file in an IICS export package. -type exportMetadata struct { - Name string `json:"name"` - SourceOrgID string `json:"sourceOrgId"` - SourceOrgName string `json:"sourceOrgName"` - Tags []exportTag `json:"tags,omitempty"` - ExportedObjects []exportedObject `json:"exportedObjects"` -} - -type exportTag struct { - ID string `json:"id"` - Name string `json:"name"` - Description *string `json:"description"` -} - -// exportedObject is one asset record in exportMetadata.v2.json. -type exportedObject struct { - ObjectGUID string `json:"objectGuid"` - ObjectName string `json:"objectName"` - ObjectType string `json:"objectType"` - Path string `json:"path"` - ProviderName json.RawMessage `json:"providerName,omitempty"` - Metadata json.RawMessage `json:"metadata"` -} - -func exportedObjectsToManifestLog(objects []exportedObject) []release.ManifestLogAsset { - rows := make([]release.ManifestLogAsset, 0, len(objects)) - for _, object := range objects { - rows = append(rows, release.ManifestLogAsset{ - ID: object.ObjectGUID, - Type: object.ObjectType, - Path: object.Path, - }) - } - return rows -} - -// objectRefs extracts metadata.objectRefs while preserving all other metadata fields. -func (o exportedObject) objectRefs() []string { - metadata := bytes.TrimSpace(o.Metadata) - if len(metadata) == 0 || bytes.Equal(metadata, []byte("null")) { - return nil - } - var m struct { - ObjectRefs []string `json:"objectRefs"` - } - if err := json.Unmarshal(metadata, &m); err != nil { - return nil - } - return append([]string(nil), m.ObjectRefs...) -} - -// setObjectRefs updates metadata.objectRefs without dropping other metadata fields. -func (o *exportedObject) setObjectRefs(refs []string) error { - metadata := bytes.TrimSpace(o.Metadata) - m := make(map[string]json.RawMessage) - if len(metadata) > 0 && !bytes.Equal(metadata, []byte("null")) { - if err := json.Unmarshal(metadata, &m); err != nil { - return fmt.Errorf("parsing metadata for %s: %w", o.ObjectGUID, err) - } - } - rawRefs, err := json.Marshal(refs) - if err != nil { - return fmt.Errorf("serializing metadata refs for %s: %w", o.ObjectGUID, err) - } - m["objectRefs"] = rawRefs - updated, err := json.Marshal(m) - if err != nil { - return fmt.Errorf("serializing metadata for %s: %w", o.ObjectGUID, err) - } - o.Metadata = updated - return nil -} +// exportMetadata and exportedObject are aliases for the corresponding +// internal/packaging types (moved there as part of the selective-export +// orchestration pipeline). Aliasing keeps every existing call site in this file +// (readExportMetadata, resolveDependencies, package dependencies, etc.) source +// compatible without behavioral change. +type exportMetadata = packaging.ExportMetadata +type exportedObject = packaging.ExportedObject // dependencyItem is one row in the dependency output. type dependencyItem struct { @@ -1173,41 +812,7 @@ type dependencyEdge struct { // readExportMetadata reads exportMetadata.v2.json from a ZIP file or workspace directory. func readExportMetadata(filePath, workspace string) (*exportMetadata, error) { - if workspace != "" { - data, err := os.ReadFile(filepath.Join(workspace, "exportMetadata.v2.json")) - if err != nil { - return nil, fmt.Errorf("reading exportMetadata.v2.json: %w", err) - } - var meta exportMetadata - if err := json.Unmarshal(data, &meta); err != nil { - return nil, fmt.Errorf("parsing exportMetadata.v2.json: %w", err) - } - return &meta, nil - } - r, err := zip.OpenReader(filePath) - if err != nil { - return nil, fmt.Errorf("opening package file: %w", err) - } - defer func() { _ = r.Close() }() - for _, f := range r.File { - if f.Name == "exportMetadata.v2.json" { - rc, oErr := f.Open() - if oErr != nil { - return nil, fmt.Errorf("opening exportMetadata.v2.json in ZIP: %w", oErr) - } - data, rErr := io.ReadAll(rc) - _ = rc.Close() - if rErr != nil { - return nil, fmt.Errorf("reading exportMetadata.v2.json from ZIP: %w", rErr) - } - var meta exportMetadata - if err := json.Unmarshal(data, &meta); err != nil { - return nil, fmt.Errorf("parsing exportMetadata.v2.json: %w", err) - } - return &meta, nil - } - } - return nil, fmt.Errorf("exportMetadata.v2.json not found in package") + return packaging.ReadExportMetadata(filePath, workspace) } func readExportChecksumEntries(filePath, workspace string) (map[string]bool, error) { @@ -1351,7 +956,7 @@ func resolveDependencies( } } - for _, refGUID := range obj.objectRefs() { + for _, refGUID := range obj.ObjectRefs() { rawEdges = append(rawEdges, [2]string{guid, refGUID}) if refGUID != "" && !visited[refGUID] { queue = append(queue, refGUID) diff --git a/internal/packaging/build.go b/internal/packaging/build.go new file mode 100644 index 0000000..ef4fffa --- /dev/null +++ b/internal/packaging/build.go @@ -0,0 +1,399 @@ +package packaging + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/jbrazda/iics-cli/internal/client" + "github.com/jbrazda/iics-cli/internal/dependencies" + "github.com/jbrazda/iics-cli/internal/release" +) + +// BuildOptions configures the selective-export pipeline run by BuildSelectivePackage. +type BuildOptions struct { + // ExcludeFoundTransitive mirrors the --exclude-found-transitive flag. + ExcludeFoundTransitive bool + // PackageName mirrors the --name flag override. + PackageName string + // IncludeTags mirrors the --include-tags flag. + IncludeTags bool + // Target is the output ZIP path (--target); its basename (without extension) is + // used to derive the default package name when PackageName is empty, same as today. + Target string +} + +// Result carries everything cmd/package.go needs to write output/log a "package create" run. +type Result struct { + // Files holds the final package contents (excluding exportPackage.chksum), including + // the regenerated exportMetadata.v2.json and ContentsofExportPackage_.csv. + Files map[string][]byte + // ReportIncluded is the set of selected assets for the manifest log. + ReportIncluded []release.ManifestLogAsset + // ReportSelected is the count of selected assets. + ReportSelected int + // ReportExcluded is the count of transitive-found rows excluded by + // --exclude-found-transitive (manifest rows + closure-suppressed). + ReportExcluded int + // Warnings are selection warnings (e.g. duplicate manifest selectors) to be printed + // to stderr as "Warning: ", unconditionally (not gated by --verbose). + Warnings []string + // Notices are verbose-only progress lines, in emission order, so cmd/ can decide + // whether to print them under --verbose. Each entry includes its trailing newline. + Notices []string +} + +// BuildSelectivePackage runs the full selective-export pipeline (selection -> parent +// container / referenced-closure / CDI-ref inclusion -> metadata graph -> file +// filtering -> exportMetadata.v2.json / ContentsofExportPackage CSV regeneration) +// over already-collected file contents from a source directory. +// +// manifestEntries/manifestStats must come from parsing the selection manifest +// (see cmd's readPackageSelectionManifest); the caller decides whether a selection +// manifest was supplied at all and only calls this when it was. +func BuildSelectivePackage( + fileContents map[string][]byte, + absSource string, + manifestEntries []client.ArtifactEntry, + manifestStats dependencies.BuildManifestStats, + opts BuildOptions, +) (Result, error) { + var result Result + + if len(manifestEntries) == 0 { + return result, fmt.Errorf("selection manifest is empty") + } + + meta, metaErr := ReadExportMetadata("", absSource) + if metaErr != nil { + return result, fmt.Errorf("reading source metadata for selective packaging: %w", metaErr) + } + + exported := make([]dependencies.ExportedObjectRef, 0, len(meta.ExportedObjects)) + for _, o := range meta.ExportedObjects { + exported = append(exported, dependencies.ExportedObjectRef{ + ObjectGUID: o.ObjectGUID, + ObjectName: o.ObjectName, + ObjectType: o.ObjectType, + Path: o.Path, + }) + } + selectedIDs, warnings, selErr := dependencies.SelectExportedObjects(manifestEntries, exported) + if selErr != nil { + return result, selErr + } + parentAdded := dependencies.IncludeParentContainers(exported, selectedIDs) + closureNodes := make([]dependencies.RefClosureNode, 0, len(meta.ExportedObjects)) + for _, o := range meta.ExportedObjects { + if o.ObjectGUID == "" { + continue + } + closureNodes = append(closureNodes, dependencies.RefClosureNode{ + ID: o.ObjectGUID, + Refs: o.ObjectRefs(), + }) + } + excludedSelectedIDs := make(map[string]bool) + if opts.ExcludeFoundTransitive && len(manifestStats.ExcludedEntries) > 0 { + resolvedExcluded, _, excludedSelErr := dependencies.SelectExportedObjects(manifestStats.ExcludedEntries, exported) + if excludedSelErr != nil { + return result, fmt.Errorf("resolving excluded transitive-found entries: %w", excludedSelErr) + } + excludedSelectedIDs = resolvedExcluded + } + + closureAdded := 0 + closureSuppressedExcluded := 0 + if !opts.ExcludeFoundTransitive { + closureAdded = dependencies.IncludeReferencedClosure(closureNodes, selectedIDs) + } else { + closureAddedIDs := dependencies.AddedIDsAfterClosure(closureNodes, selectedIDs) + closureSuppressedExcluded = len(closureAddedIDs) + if len(excludedSelectedIDs) > 0 && len(closureAddedIDs) > 0 { + // Keep a floor count based on all closure-suppressed additions and + // use excluded-entry overlap to avoid undercounting when manifest rows + // and closure additions are both present. + overlap := dependencies.CountSetIntersection(closureAddedIDs, excludedSelectedIDs) + if overlap > closureSuppressedExcluded { + closureSuppressedExcluded = overlap + } + } + } + if len(selectedIDs) == 0 { + return result, fmt.Errorf("no assets matched selection manifest") + } + + result.Warnings = warnings + + if opts.ExcludeFoundTransitive { + totalExcluded := manifestStats.ExcludedTransitiveFound + closureSuppressedExcluded + result.Notices = append(result.Notices, fmt.Sprintf( + "Selection filter: excluded %d transitive found rows using %s (manifest rows: %d, closure-suppressed: %d)\n", + totalExcluded, + manifestStats.SelectedStatusColumnName, + manifestStats.ExcludedTransitiveFound, + closureSuppressedExcluded, + )) + } + if parentAdded > 0 { + result.Notices = append(result.Notices, fmt.Sprintf("Selection refinement: included %d inferred parent Project/Folder objects\n", parentAdded)) + } + if closureAdded > 0 { + result.Notices = append(result.Notices, fmt.Sprintf("Selection refinement: included %d in-package referenced dependencies\n", closureAdded)) + } + cdiRefAdded := 0 + if opts.ExcludeFoundTransitive { + refNodes := make([]dependencies.CDIObjectRefNode, 0, len(meta.ExportedObjects)) + for _, o := range meta.ExportedObjects { + if o.ObjectGUID == "" { + continue + } + refNodes = append(refNodes, dependencies.CDIObjectRefNode{ + ID: o.ObjectGUID, + Type: o.ObjectType, + ObjectRefs: o.ObjectRefs(), + }) + } + cdiRefAdded = dependencies.IncludeCDISysRefsFromObjectRefs(refNodes, selectedIDs) + if cdiRefAdded > 0 { + result.Notices = append(result.Notices, fmt.Sprintf("Selection refinement: included %d CDI Connection/AgentGroup refs from objectRefs\n", cdiRefAdded)) + } + } + + selectedObjects := make([]ExportedObject, 0, len(selectedIDs)) + for _, o := range meta.ExportedObjects { + if selectedIDs[o.ObjectGUID] { + selectedObjects = append(selectedObjects, o) + } + } + if len(selectedObjects) == 0 { + return result, fmt.Errorf("selection manifest resolved no exported objects") + } + result.ReportIncluded = ExportedObjectsToManifestLog(selectedObjects) + result.ReportSelected = len(selectedObjects) + result.ReportExcluded = manifestStats.ExcludedTransitiveFound + closureSuppressedExcluded + metadataObjects := selectedObjects + if !opts.ExcludeFoundTransitive { + selectedNodes := make([]dependencies.ObjectRefsNode, len(selectedObjects)) + for i, o := range selectedObjects { + selectedNodes[i] = dependencies.ObjectRefsNode{ + ID: o.ObjectGUID, + ObjectRefs: o.ObjectRefs(), + } + } + prunedRefsByID, prunedCount := dependencies.PruneDanglingObjectRefs(selectedNodes) + for i := range selectedObjects { + if setErr := selectedObjects[i].SetObjectRefs(prunedRefsByID[selectedObjects[i].ObjectGUID]); setErr != nil { + return result, setErr + } + } + postPruneNodes := make([]dependencies.ObjectRefsNode, len(selectedObjects)) + for i, o := range selectedObjects { + postPruneNodes[i] = dependencies.ObjectRefsNode{ + ID: o.ObjectGUID, + ObjectRefs: o.ObjectRefs(), + } + } + if dangling := dependencies.CountDanglingObjectRefs(postPruneNodes); dangling > 0 { + return result, fmt.Errorf("selection produced %d unresolved objectRefs after pruning; cannot create import-safe package", dangling) + } + if prunedCount > 0 { + result.Notices = append(result.Notices, fmt.Sprintf("Selection refinement: pruned %d dangling metadata objectRefs\n", prunedCount)) + } + } else { + metadataIDs := buildMetadataGraphForSelection(meta.ExportedObjects, selectedIDs, excludedSelectedIDs) + metadataObjectList := make([]ExportedObject, 0, len(metadataIDs)) + for _, o := range meta.ExportedObjects { + if metadataIDs[o.ObjectGUID] { + metadataObjectList = append(metadataObjectList, o) + } + } + if len(metadataObjectList) == 0 { + return result, fmt.Errorf("selection metadata graph resolved no exported objects") + } + metadataNodes := make([]dependencies.ObjectRefsNode, len(metadataObjectList)) + for i, o := range metadataObjectList { + metadataNodes[i] = dependencies.ObjectRefsNode{ + ID: o.ObjectGUID, + ObjectRefs: o.ObjectRefs(), + } + } + prunedRefsByID, _ := dependencies.PruneDanglingObjectRefs(metadataNodes) + for i := range metadataObjectList { + if setErr := metadataObjectList[i].SetObjectRefs(prunedRefsByID[metadataObjectList[i].ObjectGUID]); setErr != nil { + return result, setErr + } + } + metadataObjects = metadataObjectList + } + + filtered := filterPackageFilesForSelection(fileContents, meta.ExportedObjects, selectedIDs) + if len(filtered) == 0 { + return result, fmt.Errorf("no package files remained after selection filtering") + } + + finalPackageName := opts.PackageName + if finalPackageName == "" { + finalPackageName = strings.TrimSuffix(filepath.Base(opts.Target), filepath.Ext(opts.Target)) + } + + meta.Name = finalPackageName + if !opts.IncludeTags { + meta.Tags = nil + } + meta.ExportedObjects = metadataObjects + metaData, marshalErr := json.MarshalIndent(meta, "", " ") + if marshalErr != nil { + return result, fmt.Errorf("serializing filtered exportMetadata.v2.json: %w", marshalErr) + } + filtered["exportMetadata.v2.json"] = metaData + + contentsCSV, csvErr := buildContentsOfExportPackageCSV(selectedObjects) + if csvErr != nil { + return result, csvErr + } + filtered["ContentsofExportPackage_"+finalPackageName+".csv"] = contentsCSV + + result.Files = filtered + return result, nil +} + +// filterPackageFilesForSelection drops files backing exported objects that were not +// selected, and always drops any preexisting exportPackage.chksum, exportMetadata.v2.json, +// and ContentsofExportPackage_*.csv entries (these are regenerated by the caller). +func filterPackageFilesForSelection( + fileContents map[string][]byte, + allObjects []ExportedObject, + selectedIDs map[string]bool, +) map[string][]byte { + allObjectFiles := make(map[string]bool) + selectedObjectFiles := make(map[string]bool) + for _, o := range allObjects { + candidates := dependencies.ObjectChecksumCandidates(o.Path, o.ObjectName, o.ObjectType) + for _, c := range candidates { + if _, ok := fileContents[c]; ok { + allObjectFiles[c] = true + if selectedIDs[o.ObjectGUID] { + selectedObjectFiles[c] = true + } + } + } + } + + out := make(map[string][]byte) + for path, data := range fileContents { + if path == "exportPackage.chksum" || path == "exportMetadata.v2.json" { + continue + } + if strings.HasPrefix(path, "ContentsofExportPackage_") && strings.HasSuffix(path, ".csv") { + continue + } + if allObjectFiles[path] { + if selectedObjectFiles[path] { + out[path] = data + } + continue + } + out[path] = data + } + return out +} + +// buildMetadataGraphForSelection computes the set of exported-object IDs that must +// remain in the regenerated exportMetadata.v2.json for an --exclude-found-transitive +// run: the selected IDs plus their in-package objectRefs closure, minus any excluded +// (transitive-found) IDs that were not themselves explicitly selected. +func buildMetadataGraphForSelection( + allObjects []ExportedObject, + selectedIDs map[string]bool, + excludedIDs map[string]bool, +) map[string]bool { + metadataIDs := make(map[string]bool, len(selectedIDs)) + for id := range selectedIDs { + metadataIDs[id] = true + } + byID := make(map[string]ExportedObject, len(allObjects)) + exportedRefs := make([]dependencies.ExportedObjectRef, 0, len(allObjects)) + for _, o := range allObjects { + if o.ObjectGUID == "" { + continue + } + byID[o.ObjectGUID] = o + exportedRefs = append(exportedRefs, dependencies.ExportedObjectRef{ + ObjectGUID: o.ObjectGUID, + ObjectName: o.ObjectName, + ObjectType: o.ObjectType, + Path: o.Path, + }) + } + + queue := make([]string, 0, len(metadataIDs)) + for id := range metadataIDs { + queue = append(queue, id) + } + for len(queue) > 0 { + id := queue[0] + queue = queue[1:] + obj, ok := byID[id] + if !ok { + continue + } + for _, refID := range obj.ObjectRefs() { + if excludedIDs[refID] && !selectedIDs[refID] { + continue + } + if _, exists := byID[refID]; !exists { + continue + } + if metadataIDs[refID] { + continue + } + metadataIDs[refID] = true + queue = append(queue, refID) + } + } + + // Include container hierarchy needed for selected/closure objects. + _ = dependencies.IncludeParentContainers(exportedRefs, metadataIDs) + for id := range excludedIDs { + if selectedIDs[id] { + continue + } + delete(metadataIDs, id) + } + return metadataIDs +} + +// buildContentsOfExportPackageCSV builds the ContentsofExportPackage_.csv content. +func buildContentsOfExportPackageCSV(objects []ExportedObject) ([]byte, error) { + rows := make([][]string, 0, len(objects)+1) + rows = append(rows, []string{"objectPath", "objectName", "objectType", "id"}) + + sorted := make([]ExportedObject, len(objects)) + copy(sorted, objects) + sort.SliceStable(sorted, func(i, j int) bool { + a := client.NormalizeLocationPath(sorted[i].Path) + "/" + sorted[i].ObjectName + "." + sorted[i].ObjectType + b := client.NormalizeLocationPath(sorted[j].Path) + "/" + sorted[j].ObjectName + "." + sorted[j].ObjectType + return a < b + }) + + for _, o := range sorted { + rows = append(rows, []string{ + o.Path, + o.ObjectName, + o.ObjectType, + o.ObjectGUID, + }) + } + + var b bytes.Buffer + w := csv.NewWriter(&b) + if err := w.WriteAll(rows); err != nil { + return nil, fmt.Errorf("writing ContentsofExportPackage csv: %w", err) + } + return b.Bytes(), nil +} diff --git a/internal/packaging/build_test.go b/internal/packaging/build_test.go new file mode 100644 index 0000000..6cb3c21 --- /dev/null +++ b/internal/packaging/build_test.go @@ -0,0 +1,309 @@ +package packaging + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jbrazda/iics-cli/internal/client" + "github.com/jbrazda/iics-cli/internal/dependencies" +) + +// writeSourceMetadata writes exportMetadata.v2.json into dir so ReadExportMetadata("", dir) +// (called internally by BuildSelectivePackage) can read it back. +func writeSourceMetadata(t *testing.T, dir string, meta ExportMetadata) { + t.Helper() + data, err := json.Marshal(meta) + if err != nil { + t.Fatalf("marshal fixture metadata: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "exportMetadata.v2.json"), data, 0o644); err != nil { + t.Fatalf("writing fixture exportMetadata.v2.json: %v", err) + } +} + +func rawRefs(t *testing.T, refs ...string) json.RawMessage { + t.Helper() + data, err := json.Marshal(map[string][]string{"objectRefs": refs}) + if err != nil { + t.Fatalf("marshal refs: %v", err) + } + return data +} + +// baseFixtureObjects returns a small package graph: +// +// proj (Project /Explore) -> App +// folder(Folder /Explore/App) -> Mappings +// obj1 (DTEMPLATE .../Mappings) -> m1, refs conn1 +// conn1 (Connection /SYS/Connections) -> ConnA +// obj2 (DTEMPLATE .../Mappings) -> m2, unrelated, no refs +func baseFixtureObjects(t *testing.T) []ExportedObject { + t.Helper() + return []ExportedObject{ + {ObjectGUID: "proj", ObjectName: "App", ObjectType: "Project", Path: "/Explore"}, + {ObjectGUID: "folder", ObjectName: "Mappings", ObjectType: "Folder", Path: "/Explore/App"}, + {ObjectGUID: "obj1", ObjectName: "m1", ObjectType: "DTEMPLATE", Path: "/Explore/App/Mappings", Metadata: rawRefs(t, "conn1")}, + {ObjectGUID: "conn1", ObjectName: "ConnA", ObjectType: "Connection", Path: "/SYS/Connections"}, + {ObjectGUID: "obj2", ObjectName: "m2", ObjectType: "DTEMPLATE", Path: "/Explore/App/Mappings"}, + } +} + +func candidateFile(path, name, typ string) string { + return strings.TrimPrefix(strings.TrimSuffix(path, "/"), "/") + "/" + name + "." + typ + ".xml" +} + +func baseFixtureFiles() map[string][]byte { + return map[string][]byte{ + candidateFile("/Explore/App/Mappings", "m1", "DTEMPLATE"): []byte("obj1-data"), + candidateFile("/SYS/Connections", "ConnA", "Connection"): []byte("conn1-data"), + candidateFile("/Explore/App/Mappings", "m2", "DTEMPLATE"): []byte("obj2-data"), + "readme.txt": []byte("not an object file"), + } +} + +func TestBuildSelectivePackage_DefaultClosureAndParents(t *testing.T) { + dir := t.TempDir() + writeSourceMetadata(t, dir, ExportMetadata{Name: "src", ExportedObjects: baseFixtureObjects(t)}) + fileContents := baseFixtureFiles() + + entries := []client.ArtifactEntry{{Path: "Explore/App/Mappings/m1", Type: "DTEMPLATE"}} + + result, err := BuildSelectivePackage(fileContents, dir, entries, dependencies.BuildManifestStats{}, BuildOptions{ + Target: "/out/mypkg.zip", + }) + if err != nil { + t.Fatalf("BuildSelectivePackage() error: %v", err) + } + + if result.ReportSelected != 4 { + t.Fatalf("ReportSelected = %d, want 4 (obj1, folder, proj, conn1)", result.ReportSelected) + } + if result.ReportExcluded != 0 { + t.Fatalf("ReportExcluded = %d, want 0", result.ReportExcluded) + } + if len(result.Warnings) != 0 { + t.Fatalf("unexpected warnings: %v", result.Warnings) + } + + // obj1 and conn1 files retained, obj2's file dropped, unrelated file untouched. + if _, ok := result.Files[candidateFile("/Explore/App/Mappings", "m1", "DTEMPLATE")]; !ok { + t.Fatal("expected obj1 file to be retained") + } + if _, ok := result.Files[candidateFile("/SYS/Connections", "ConnA", "Connection")]; !ok { + t.Fatal("expected conn1 file to be retained (closure)") + } + if _, ok := result.Files[candidateFile("/Explore/App/Mappings", "m2", "DTEMPLATE")]; ok { + t.Fatal("expected obj2 file to be dropped (not selected)") + } + if _, ok := result.Files["readme.txt"]; !ok { + t.Fatal("expected unrelated file to be retained untouched") + } + + // exportMetadata.v2.json regenerated with the default name derived from --target basename. + metaBytes, ok := result.Files["exportMetadata.v2.json"] + if !ok { + t.Fatal("expected regenerated exportMetadata.v2.json") + } + var meta ExportMetadata + if err := json.Unmarshal(metaBytes, &meta); err != nil { + t.Fatalf("unmarshal regenerated metadata: %v", err) + } + if meta.Name != "mypkg" { + t.Fatalf("meta.Name = %q, want %q (derived from --target)", meta.Name, "mypkg") + } + if len(meta.ExportedObjects) != 4 { + t.Fatalf("regenerated metadata has %d objects, want 4", len(meta.ExportedObjects)) + } + + // ContentsofExportPackage CSV regenerated using the derived name. + if _, ok := result.Files["ContentsofExportPackage_mypkg.csv"]; !ok { + t.Fatal("expected ContentsofExportPackage_mypkg.csv to be generated") + } + + // Notices: parent-container inclusion (folder+proj) then closure inclusion (conn1). + if len(result.Notices) != 2 { + t.Fatalf("Notices = %v, want 2 entries", result.Notices) + } + if !strings.Contains(result.Notices[0], "included 2 inferred parent Project/Folder objects") { + t.Fatalf("Notices[0] = %q, want parent-container notice", result.Notices[0]) + } + if !strings.Contains(result.Notices[1], "included 1 in-package referenced dependencies") { + t.Fatalf("Notices[1] = %q, want closure notice", result.Notices[1]) + } +} + +func TestBuildSelectivePackage_ExcludeFoundTransitiveKeepsCDIRefs(t *testing.T) { + dir := t.TempDir() + writeSourceMetadata(t, dir, ExportMetadata{Name: "src", ExportedObjects: baseFixtureObjects(t)}) + fileContents := baseFixtureFiles() + + entries := []client.ArtifactEntry{{Path: "Explore/App/Mappings/m1", Type: "DTEMPLATE"}} + stats := dependencies.BuildManifestStats{ + ExcludedTransitiveFound: 3, + SelectedStatusColumnName: "STATUS (qa)", + } + + result, err := BuildSelectivePackage(fileContents, dir, entries, stats, BuildOptions{ + ExcludeFoundTransitive: true, + PackageName: "explicit-name", + Target: "/out/mypkg.zip", + }) + if err != nil { + t.Fatalf("BuildSelectivePackage() error: %v", err) + } + + // conn1 is re-included via CDI ref rescue even though ordinary closure is suppressed. + if result.ReportSelected != 4 { + t.Fatalf("ReportSelected = %d, want 4", result.ReportSelected) + } + // closureSuppressedExcluded should count the 1 suppressed closure addition (conn1). + if result.ReportExcluded != 4 { + t.Fatalf("ReportExcluded = %d, want 4 (3 manifest-excluded + 1 closure-suppressed)", result.ReportExcluded) + } + + if _, ok := result.Files[candidateFile("/SYS/Connections", "ConnA", "Connection")]; !ok { + t.Fatal("expected conn1 file to be retained via CDI ref rescue") + } + + // PackageName override is honored over the --target-derived default. + if _, ok := result.Files["ContentsofExportPackage_explicit-name.csv"]; !ok { + t.Fatal("expected ContentsofExportPackage_explicit-name.csv") + } + + joined := strings.Join(result.Notices, "|") + if !strings.Contains(joined, "Selection filter: excluded 4 transitive found rows using STATUS (qa) (manifest rows: 3, closure-suppressed: 1)") { + t.Fatalf("expected exclude-transitive notice, got: %v", result.Notices) + } + if !strings.Contains(joined, "included 1 CDI Connection/AgentGroup refs from objectRefs") { + t.Fatalf("expected CDI ref notice, got: %v", result.Notices) + } + if strings.Contains(joined, "in-package referenced dependencies") { + t.Fatalf("did not expect ordinary closure notice when --exclude-found-transitive suppresses it, got: %v", result.Notices) + } +} + +func TestBuildSelectivePackage_EmptyManifestErrors(t *testing.T) { + dir := t.TempDir() + _, err := BuildSelectivePackage(map[string][]byte{}, dir, nil, dependencies.BuildManifestStats{}, BuildOptions{}) + if err == nil || !strings.Contains(err.Error(), "selection manifest is empty") { + t.Fatalf("expected 'selection manifest is empty' error, got: %v", err) + } +} + +func TestBuildSelectivePackage_NoFilesRemainAfterFiltering(t *testing.T) { + dir := t.TempDir() + writeSourceMetadata(t, dir, ExportMetadata{Name: "src", ExportedObjects: baseFixtureObjects(t)}) + + // Only obj2's file exists on disk, and the manifest selects obj1 only, so + // filtering drops obj2's file and every synthetic package file is regenerated + // from scratch, leaving nothing behind for filterPackageFilesForSelection to keep. + fileContents := map[string][]byte{ + candidateFile("/Explore/App/Mappings", "m2", "DTEMPLATE"): []byte("obj2-data"), + "exportPackage.chksum": []byte("stale"), + "exportMetadata.v2.json": []byte("stale"), + "ContentsofExportPackage_x.csv": []byte("stale"), + } + entries := []client.ArtifactEntry{{Path: "Explore/App/Mappings/m1", Type: "DTEMPLATE"}} + + _, err := BuildSelectivePackage(fileContents, dir, entries, dependencies.BuildManifestStats{}, BuildOptions{Target: "/out/p.zip"}) + if err == nil || !strings.Contains(err.Error(), "no package files remained after selection filtering") { + t.Fatalf("expected 'no package files remained' error, got: %v", err) + } +} + +func TestBuildSelectivePackage_DuplicateManifestSelectorWarns(t *testing.T) { + dir := t.TempDir() + writeSourceMetadata(t, dir, ExportMetadata{Name: "src", ExportedObjects: baseFixtureObjects(t)}) + fileContents := baseFixtureFiles() + + entries := []client.ArtifactEntry{ + {Path: "Explore/App/Mappings/m1", Type: "DTEMPLATE"}, + {Path: "Explore/App/Mappings/m1", Type: "DTEMPLATE"}, + } + + result, err := BuildSelectivePackage(fileContents, dir, entries, dependencies.BuildManifestStats{}, BuildOptions{Target: "/out/p.zip"}) + if err != nil { + t.Fatalf("BuildSelectivePackage() error: %v", err) + } + if len(result.Warnings) == 0 { + t.Fatal("expected a duplicate-manifest-selector warning") + } +} + +func TestFilterPackageFilesForSelection(t *testing.T) { + objects := baseFixtureObjects(t) + selected := map[string]bool{"obj1": true} + files := baseFixtureFiles() + + out := filterPackageFilesForSelection(files, objects, selected) + + if _, ok := out[candidateFile("/Explore/App/Mappings", "m1", "DTEMPLATE")]; !ok { + t.Fatal("expected selected object's file retained") + } + if _, ok := out[candidateFile("/Explore/App/Mappings", "m2", "DTEMPLATE")]; ok { + t.Fatal("expected unselected object's file dropped") + } + if _, ok := out["readme.txt"]; !ok { + t.Fatal("expected non-object file retained") + } +} + +func TestFilterPackageFilesForSelection_AlwaysDropsRegeneratedEntries(t *testing.T) { + files := map[string][]byte{ + "exportPackage.chksum": []byte("x"), + "exportMetadata.v2.json": []byte("x"), + "ContentsofExportPackage_a.csv": []byte("x"), + "keep.txt": []byte("x"), + } + out := filterPackageFilesForSelection(files, nil, nil) + if len(out) != 1 { + t.Fatalf("expected only keep.txt to remain, got %v", out) + } + if _, ok := out["keep.txt"]; !ok { + t.Fatal("expected keep.txt retained") + } +} + +func TestBuildContentsOfExportPackageCSV(t *testing.T) { + objects := []ExportedObject{ + {ObjectGUID: "b", ObjectName: "zeta", ObjectType: "PROCESS", Path: "/Explore/App"}, + {ObjectGUID: "a", ObjectName: "alpha", ObjectType: "PROCESS", Path: "/Explore/App"}, + } + data, err := buildContentsOfExportPackageCSV(objects) + if err != nil { + t.Fatalf("buildContentsOfExportPackageCSV() error: %v", err) + } + lines := strings.Split(strings.TrimRight(string(data), "\r\n"), "\n") + if len(lines) != 3 { + t.Fatalf("expected header + 2 rows, got %d lines: %v", len(lines), lines) + } + if lines[0] != "objectPath,objectName,objectType,id" { + t.Fatalf("unexpected header: %q", lines[0]) + } + // alpha sorts before zeta. + if !strings.Contains(lines[1], "alpha") { + t.Fatalf("expected alpha row first, got: %v", lines) + } + if !strings.Contains(lines[2], "zeta") { + t.Fatalf("expected zeta row second, got: %v", lines) + } +} + +func TestBuildMetadataGraphForSelection(t *testing.T) { + objects := baseFixtureObjects(t) + selected := map[string]bool{"obj1": true} + + ids := buildMetadataGraphForSelection(objects, selected, nil) + + for _, want := range []string{"obj1", "conn1"} { + if !ids[want] { + t.Fatalf("expected %q included in metadata graph, got %v", want, ids) + } + } + if ids["obj2"] { + t.Fatal("did not expect unrelated obj2 in metadata graph") + } +} diff --git a/internal/packaging/metadata.go b/internal/packaging/metadata.go new file mode 100644 index 0000000..a468cc9 --- /dev/null +++ b/internal/packaging/metadata.go @@ -0,0 +1,51 @@ +package packaging + +import ( + "archive/zip" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" +) + +// ReadExportMetadata reads exportMetadata.v2.json from a ZIP file (filePath) or an +// expanded workspace directory (workspace). Exactly one of filePath/workspace should +// be non-empty; workspace takes precedence when both are set. +func ReadExportMetadata(filePath, workspace string) (*ExportMetadata, error) { + if workspace != "" { + data, err := os.ReadFile(filepath.Join(workspace, "exportMetadata.v2.json")) + if err != nil { + return nil, fmt.Errorf("reading exportMetadata.v2.json: %w", err) + } + var meta ExportMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return nil, fmt.Errorf("parsing exportMetadata.v2.json: %w", err) + } + return &meta, nil + } + r, err := zip.OpenReader(filePath) + if err != nil { + return nil, fmt.Errorf("opening package file: %w", err) + } + defer func() { _ = r.Close() }() + for _, f := range r.File { + if f.Name == "exportMetadata.v2.json" { + rc, oErr := f.Open() + if oErr != nil { + return nil, fmt.Errorf("opening exportMetadata.v2.json in ZIP: %w", oErr) + } + data, rErr := io.ReadAll(rc) + _ = rc.Close() + if rErr != nil { + return nil, fmt.Errorf("reading exportMetadata.v2.json from ZIP: %w", rErr) + } + var meta ExportMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return nil, fmt.Errorf("parsing exportMetadata.v2.json: %w", err) + } + return &meta, nil + } + } + return nil, fmt.Errorf("exportMetadata.v2.json not found in package") +} diff --git a/internal/packaging/metadata_test.go b/internal/packaging/metadata_test.go new file mode 100644 index 0000000..a2f97dd --- /dev/null +++ b/internal/packaging/metadata_test.go @@ -0,0 +1,89 @@ +package packaging + +import ( + "archive/zip" + "os" + "path/filepath" + "testing" +) + +func TestReadExportMetadata_Workspace(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "exportMetadata.v2.json"), `{"name":"pkg","exportedObjects":[{"objectGuid":"g1","objectName":"m1","objectType":"DTEMPLATE","path":"/Explore/App"}]}`) + + meta, err := ReadExportMetadata("", dir) + if err != nil { + t.Fatalf("ReadExportMetadata() error: %v", err) + } + if meta.Name != "pkg" || len(meta.ExportedObjects) != 1 || meta.ExportedObjects[0].ObjectGUID != "g1" { + t.Fatalf("unexpected metadata: %+v", meta) + } +} + +func TestReadExportMetadata_WorkspaceMissingFile(t *testing.T) { + dir := t.TempDir() + if _, err := ReadExportMetadata("", dir); err == nil { + t.Fatal("expected error when exportMetadata.v2.json is missing") + } +} + +func TestReadExportMetadata_ZIP(t *testing.T) { + dir := t.TempDir() + zipPath := filepath.Join(dir, "pkg.zip") + f, err := os.Create(zipPath) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(f) + w, err := zw.Create("exportMetadata.v2.json") + if err != nil { + t.Fatal(err) + } + if _, werr := w.Write([]byte(`{"name":"zippkg","exportedObjects":[]}`)); werr != nil { + t.Fatal(werr) + } + if cerr := zw.Close(); cerr != nil { + t.Fatal(cerr) + } + if cerr := f.Close(); cerr != nil { + t.Fatal(cerr) + } + + meta, err := ReadExportMetadata(zipPath, "") + if err != nil { + t.Fatalf("ReadExportMetadata() error: %v", err) + } + if meta.Name != "zippkg" { + t.Fatalf("unexpected metadata: %+v", meta) + } +} + +func TestReadExportMetadata_ZIPMissingEntry(t *testing.T) { + dir := t.TempDir() + zipPath := filepath.Join(dir, "pkg.zip") + f, err := os.Create(zipPath) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(f) + if _, err := zw.Create("other.txt"); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + if _, err := ReadExportMetadata(zipPath, ""); err == nil { + t.Fatal("expected error when exportMetadata.v2.json entry is missing from ZIP") + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("writing %s: %v", path, err) + } +} diff --git a/internal/packaging/types.go b/internal/packaging/types.go new file mode 100644 index 0000000..cf0d28c --- /dev/null +++ b/internal/packaging/types.go @@ -0,0 +1,94 @@ +// Package packaging implements the selective-export orchestration pipeline used +// by "iics package create": resolving a selection manifest against a source +// export's exportMetadata.v2.json, expanding container/closure/CDI dependency +// inclusion, pruning dangling references, and rebuilding the filtered package +// file set (exportMetadata.v2.json + ContentsofExportPackage_.csv). +// +// This package has no Cobra or internal/output dependency so its core logic is +// unit-testable without a CLI harness or disk I/O. +package packaging + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/jbrazda/iics-cli/internal/release" +) + +// ExportMetadata represents the exportMetadata.v2.json file in an IICS export package. +type ExportMetadata struct { + Name string `json:"name"` + SourceOrgID string `json:"sourceOrgId"` + SourceOrgName string `json:"sourceOrgName"` + Tags []ExportTag `json:"tags,omitempty"` + ExportedObjects []ExportedObject `json:"exportedObjects"` +} + +// ExportTag is a root-level tag entry in exportMetadata.v2.json. +type ExportTag struct { + ID string `json:"id"` + Name string `json:"name"` + Description *string `json:"description"` +} + +// ExportedObject is one asset record in exportMetadata.v2.json. +type ExportedObject struct { + ObjectGUID string `json:"objectGuid"` + ObjectName string `json:"objectName"` + ObjectType string `json:"objectType"` + Path string `json:"path"` + ProviderName json.RawMessage `json:"providerName,omitempty"` + Metadata json.RawMessage `json:"metadata"` +} + +// ExportedObjectsToManifestLog converts exported objects into manifest log asset +// rows for release.RenderPackageCreateLog. +func ExportedObjectsToManifestLog(objects []ExportedObject) []release.ManifestLogAsset { + rows := make([]release.ManifestLogAsset, 0, len(objects)) + for _, object := range objects { + rows = append(rows, release.ManifestLogAsset{ + ID: object.ObjectGUID, + Type: object.ObjectType, + Path: object.Path, + }) + } + return rows +} + +// ObjectRefs extracts metadata.objectRefs while preserving all other metadata fields. +func (o ExportedObject) ObjectRefs() []string { + metadata := bytes.TrimSpace(o.Metadata) + if len(metadata) == 0 || bytes.Equal(metadata, []byte("null")) { + return nil + } + var m struct { + ObjectRefs []string `json:"objectRefs"` + } + if err := json.Unmarshal(metadata, &m); err != nil { + return nil + } + return append([]string(nil), m.ObjectRefs...) +} + +// SetObjectRefs updates metadata.objectRefs without dropping other metadata fields. +func (o *ExportedObject) SetObjectRefs(refs []string) error { + metadata := bytes.TrimSpace(o.Metadata) + m := make(map[string]json.RawMessage) + if len(metadata) > 0 && !bytes.Equal(metadata, []byte("null")) { + if err := json.Unmarshal(metadata, &m); err != nil { + return fmt.Errorf("parsing metadata for %s: %w", o.ObjectGUID, err) + } + } + rawRefs, err := json.Marshal(refs) + if err != nil { + return fmt.Errorf("serializing metadata refs for %s: %w", o.ObjectGUID, err) + } + m["objectRefs"] = rawRefs + updated, err := json.Marshal(m) + if err != nil { + return fmt.Errorf("serializing metadata for %s: %w", o.ObjectGUID, err) + } + o.Metadata = updated + return nil +} diff --git a/internal/packaging/types_test.go b/internal/packaging/types_test.go new file mode 100644 index 0000000..8027e82 --- /dev/null +++ b/internal/packaging/types_test.go @@ -0,0 +1,74 @@ +package packaging + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestExportedObject_ObjectRefsRoundTrip(t *testing.T) { + tests := []struct { + name string + metadata string + want []string + }{ + {name: "empty metadata", metadata: "", want: nil}, + {name: "null metadata", metadata: "null", want: nil}, + {name: "no objectRefs field", metadata: `{"foo":"bar"}`, want: nil}, + {name: "objectRefs present", metadata: `{"objectRefs":["a","b"],"other":1}`, want: []string{"a", "b"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := ExportedObject{ObjectGUID: "x", Metadata: json.RawMessage(tt.metadata)} + got := o.ObjectRefs() + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("ObjectRefs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExportedObject_SetObjectRefsPreservesOtherFields(t *testing.T) { + o := ExportedObject{ObjectGUID: "x", Metadata: json.RawMessage(`{"foo":"bar","objectRefs":["old"]}`)} + if err := o.SetObjectRefs([]string{"new1", "new2"}); err != nil { + t.Fatalf("SetObjectRefs() error: %v", err) + } + var m map[string]json.RawMessage + if err := json.Unmarshal(o.Metadata, &m); err != nil { + t.Fatalf("unmarshal updated metadata: %v", err) + } + if string(m["foo"]) != `"bar"` { + t.Fatalf("expected foo field preserved, got %s", m["foo"]) + } + var refs []string + if err := json.Unmarshal(m["objectRefs"], &refs); err != nil { + t.Fatalf("unmarshal objectRefs: %v", err) + } + if !reflect.DeepEqual(refs, []string{"new1", "new2"}) { + t.Fatalf("objectRefs = %v, want [new1 new2]", refs) + } +} + +func TestExportedObject_SetObjectRefsInvalidMetadataErrors(t *testing.T) { + o := ExportedObject{ObjectGUID: "x", Metadata: json.RawMessage(`[1,2,3]`)} + if err := o.SetObjectRefs([]string{"a"}); err == nil { + t.Fatal("expected error for metadata that is not a JSON object") + } +} + +func TestExportedObjectsToManifestLog(t *testing.T) { + objects := []ExportedObject{ + {ObjectGUID: "g1", ObjectType: "PROCESS", Path: "/Explore/App"}, + {ObjectGUID: "g2", ObjectType: "Folder", Path: "/Explore"}, + } + rows := ExportedObjectsToManifestLog(objects) + if len(rows) != 2 { + t.Fatalf("expected 2 rows, got %d", len(rows)) + } + if rows[0].ID != "g1" || rows[0].Type != "PROCESS" || rows[0].Path != "/Explore/App" { + t.Fatalf("unexpected row 0: %+v", rows[0]) + } + if rows[1].ID != "g2" || rows[1].Type != "Folder" || rows[1].Path != "/Explore" { + t.Fatalf("unexpected row 1: %+v", rows[1]) + } +} From 121d3d6411a4c066e4fd93e6c6a574ec605044d2 Mon Sep 17 00:00:00 2001 From: Jaroslav Brazda Date: Sun, 13 Sep 2026 15:12:04 -0400 Subject: [PATCH 2/5] refactor(release): collapse plan-mode duplication into BuildPlan release plan's RunE had two ~170-line branches (full-deployment vs selective/tag mode) that duplicated the same per-target pipeline (filter missing-transitive, validate, render totals, write package and publish files, accumulate manifest-log rows), differing only in how the initial asset set is resolved. Add internal/release.BuildPlan, a composing entry point over the existing per-step functions in plan.go, parameterized by asset source and a few mode-specific hooks/labels. cmd/release.go's RunE now calls it once per mode and shrinks by about a third. Drops releaseAssetsToManifestLog from cmd/manifest_log.go, dead code now that BuildPlan builds ManifestLogAsset rows internally. No flag, output, error-text, or file-format changes; no CI/docs impact. Adds tests for BuildPlan covering both modes. --- cmd/manifest_log.go | 15 -- cmd/release.go | 265 +++++++++------------------------ internal/release/build.go | 203 +++++++++++++++++++++++++ internal/release/build_test.go | 238 +++++++++++++++++++++++++++++ 4 files changed, 512 insertions(+), 209 deletions(-) create mode 100644 internal/release/build.go create mode 100644 internal/release/build_test.go diff --git a/cmd/manifest_log.go b/cmd/manifest_log.go index d130cb4..6c7ef68 100644 --- a/cmd/manifest_log.go +++ b/cmd/manifest_log.go @@ -24,18 +24,3 @@ func appendManifestLogWarning(cmd *cobra.Command, path, markdown string) { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: could not append release log %s: %v\n", path, err) } } - -func releaseAssetsToManifestLog(assets []release.Asset) []release.ManifestLogAsset { - rows := make([]release.ManifestLogAsset, 0, len(assets)) - for _, asset := range assets { - rows = append(rows, release.ManifestLogAsset{ - ID: asset.ID, - Location: asset.Location, - Type: asset.Type, - Path: asset.Path, - Dependency: asset.Dependency, - Status: asset.Status, - }) - } - return rows -} diff --git a/cmd/release.go b/cmd/release.go index ab9c7fb..eab1f78 100644 --- a/cmd/release.go +++ b/cmd/release.go @@ -6,7 +6,6 @@ import ( "io" "log/slog" "os" - "path/filepath" "strings" "time" @@ -280,103 +279,37 @@ func newReleasePlanCmd() *cobra.Command { } } - assetsByTarget := make(map[string][]release.ManifestLogAsset, len(opts.Targets)) - publishByTarget := make(map[string][]release.ManifestLogAsset, len(opts.Targets)) - filesWritten := 0 - for _, env := range opts.Targets { - envDir := filepath.Join(outputRoot, strings.ToLower(env)) - if mkErr := os.MkdirAll(envDir, 0o755); mkErr != nil { - return fmt.Errorf("creating env directory: %w", mkErr) - } - envAssets := fullAssets - if filterMissingTrans { - filtered, filterErr := release.FilterMissingTransitiveForTarget( - context.Background(), - env, - fullAssets, - targetResolutionOpts, - ) - if filterErr != nil { - return filterErr - } - envAssets = filtered - slog.Info("release plan: missing-transitive filter applied", + planResult, buildErr := release.BuildPlan(context.Background(), release.PlanOptions{ + Assets: fullAssets, + ConnectorSourceAssets: fullAssets, + Targets: opts.Targets, + FilterMissingTransitive: filterMissingTrans, + TargetResolutionOptions: targetResolutionOpts, + OutputRoot: outputRoot, + PackageFileBaseName: "full_build.package", + PlanExt: planExt, + WriteAssets: writeAssets, + PackageFields: packageFields, + PublishFields: publishFields, + IncludeConnectors: opts.IncludeConnectors, + IncludeConnections: opts.IncludeConnections, + InfoEnabled: infoEnabled, + RenderPackageTotals: makeTypeCountRenderer(logWriter), + RenderPublishTotals: makeTypeCountRenderer(logWriter), + RenderConnectorTotals: makeTypeCountRenderer(logWriter), + OnTargetFilesGenerated: func(env, packageFile, publishFile string, publishAssetCount int) { + slog.Info("release plan: full mode files generated", "environment", env, - "before", len(fullAssets), - "after", len(envAssets), + "packageFile", packageFile, + "publishFile", publishFile, + "publishAssets", publishAssetCount, ) - } - envPackageAssets, annotateErr := release.AnnotateAssetsWithTargetValidation( - context.Background(), - env, - envAssets, - targetResolutionOpts, - ) - if annotateErr != nil { - return annotateErr - } - envPackageFields := release.EnsureCurrentTargetStatusField(packageFields, env) - targetCfg := filepath.Join(envDir, "full_build.package."+planExt) - if writeErr := writeAssets(targetCfg, envPackageAssets, envPackageFields); writeErr != nil { - return writeErr - } - publishAssets := release.PublishAssets(envAssets) - publishFile := filepath.Join(envDir, "publish_assets."+planExt) - if writeErr := writeAssets(publishFile, publishAssets, publishFields); writeErr != nil { - return writeErr - } - filesWritten += 2 - if infoEnabled { - if renderErr := renderTypeCountTable( - logWriter, - fmt.Sprintf("release plan: package totals by type for %s", env), - release.AssetCountsByType(envAssets), - len(envAssets), - ); renderErr != nil { - return renderErr - } - if renderErr := renderTypeCountTable( - logWriter, - fmt.Sprintf("release plan: publish totals by type for %s", env), - release.AssetCountsByType(publishAssets), - len(publishAssets), - ); renderErr != nil { - return renderErr - } - } - assetsByTarget[env] = releaseAssetsToManifestLog(envPackageAssets) - publishByTarget[env] = releaseAssetsToManifestLog(publishAssets) - slog.Info("release plan: full mode files generated", - "environment", env, - "packageFile", targetCfg, - "publishFile", publishFile, - "publishAssets", len(publishAssets), - ) + }, + CompletedLabel: "full mode", + }) + if buildErr != nil { + return buildErr } - if release.ShouldWriteConnectorPackage(opts.IncludeConnectors, opts.IncludeConnections) { - connectorAssets := release.ConnectorPackageAssets(fullAssets) - connectorsFile := filepath.Join(outputRoot, "connectors.package."+planExt) - if writeErr := writeAssets(connectorsFile, connectorAssets, packageFields); writeErr != nil { - return writeErr - } - filesWritten++ - if infoEnabled { - if renderErr := renderTypeCountTable( - logWriter, - "release plan: connector totals by type", - release.AssetCountsByType(connectorAssets), - len(connectorAssets), - ); renderErr != nil { - return renderErr - } - } - slog.Info("release plan: connectors file generated", "connectorsFile", connectorsFile) - } - slog.Info("release plan: completed full mode", - "targets", strings.Join(opts.Targets, ","), - "outputRoot", outputRoot, - "filesWritten", filesWritten, - ) if logEnabled { appendManifestLogWarning(cmd, logPath, release.RenderReleasePlanLog(release.ReleasePlanLog{ SchemaVersion: "v1", @@ -387,8 +320,8 @@ func newReleasePlanCmd() *cobra.Command { Targets: opts.Targets, IncludeConnectors: opts.IncludeConnectors, IncludeConnections: opts.IncludeConnections, - AssetsByTarget: assetsByTarget, - PublishByTarget: publishByTarget, + AssetsByTarget: planResult.AssetsByTarget, + PublishByTarget: planResult.PublishByTarget, })) } _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Generated full-deployment plan files.") @@ -427,105 +360,40 @@ func newReleasePlanCmd() *cobra.Command { } } connectorDependencyAssets := release.ApplyPolicies(assets, true, true, excludes) - assetsByTarget := make(map[string][]release.ManifestLogAsset, len(opts.Targets)) - publishByTarget := make(map[string][]release.ManifestLogAsset, len(opts.Targets)) - filesWritten := 0 - for _, env := range opts.Targets { - slog.Info("release plan: processing target", "environment", env) - envAssets := allFiltered - if filterMissingTrans { - envFiltered, filterErr := release.FilterMissingTransitiveForTarget( - context.Background(), - env, - allFiltered, - targetResolutionOpts, - ) - if filterErr != nil { - return filterErr - } - envAssets = envFiltered - slog.Info("release plan: missing-transitive filter applied", + planResult, buildErr := release.BuildPlan(context.Background(), release.PlanOptions{ + Assets: allFiltered, + ConnectorSourceAssets: connectorDependencyAssets, + Targets: opts.Targets, + FilterMissingTransitive: filterMissingTrans, + TargetResolutionOptions: targetResolutionOpts, + OutputRoot: outputRoot, + PackageFileBaseName: "tag_build.package", + PlanExt: planExt, + WriteAssets: writeAssets, + PackageFields: packageFields, + PublishFields: publishFields, + IncludeConnectors: opts.IncludeConnectors, + IncludeConnections: opts.IncludeConnections, + InfoEnabled: infoEnabled, + RenderPackageTotals: makeTypeCountRenderer(logWriter), + RenderPublishTotals: makeTypeCountRenderer(logWriter), + RenderConnectorTotals: makeTypeCountRenderer(logWriter), + OnTargetProcessing: func(env string) { + slog.Info("release plan: processing target", "environment", env) + }, + OnTargetFilesGenerated: func(env, packageFile, publishFile string, _ int) { + slog.Info("release plan: target files generated", "environment", env, - "before", len(allFiltered), - "after", len(envAssets), + "packageFile", packageFile, + "publishFile", publishFile, ) - } - envPackageAssets, annotateErr := release.AnnotateAssetsWithTargetValidation( - context.Background(), - env, - envAssets, - targetResolutionOpts, - ) - if annotateErr != nil { - return annotateErr - } - envPackageFields := release.EnsureCurrentTargetStatusField(packageFields, env) - publishAssets := release.PublishAssets(envAssets) - if infoEnabled { - if err := renderTypeCountTable( - logWriter, - fmt.Sprintf("release plan: package totals by type for %s", env), - release.AssetCountsByType(envAssets), - len(envAssets), - ); err != nil { - return err - } - if err := renderTypeCountTable( - logWriter, - fmt.Sprintf("release plan: publish totals by type for %s", env), - release.AssetCountsByType(publishAssets), - len(publishAssets), - ); err != nil { - return err - } - } - envDir := filepath.Join(outputRoot, strings.ToLower(env)) - if err := os.MkdirAll(envDir, 0o755); err != nil { - return fmt.Errorf("creating env directory: %w", err) - } - packageFile := filepath.Join(envDir, "tag_build.package."+planExt) - publishFile := filepath.Join(envDir, "publish_assets."+planExt) - if err := writeAssets(packageFile, envPackageAssets, envPackageFields); err != nil { - return err - } - filesWritten++ - if err := writeAssets(publishFile, publishAssets, publishFields); err != nil { - return err - } - assetsByTarget[env] = releaseAssetsToManifestLog(envPackageAssets) - publishByTarget[env] = releaseAssetsToManifestLog(publishAssets) - filesWritten++ - slog.Info("release plan: target files generated", - "environment", env, - "packageFile", packageFile, - "publishFile", publishFile, - ) + }, + CompletedLabel: "selective mode", + }) + if buildErr != nil { + return buildErr } - if release.ShouldWriteConnectorPackage(opts.IncludeConnectors, opts.IncludeConnections) { - connectorAssets := release.ConnectorPackageAssets(connectorDependencyAssets) - connectorsFile := filepath.Join(outputRoot, "connectors.package."+planExt) - if err := writeAssets(connectorsFile, connectorAssets, packageFields); err != nil { - return err - } - filesWritten++ - if infoEnabled { - if err := renderTypeCountTable( - logWriter, - "release plan: connector totals by type", - release.AssetCountsByType(connectorAssets), - len(connectorAssets), - ); err != nil { - return err - } - } - slog.Info("release plan: connectors file generated", "connectorsFile", connectorsFile) - } - slog.Info("release plan: completed selective mode", - "targets", strings.Join(opts.Targets, ","), - "outputRoot", outputRoot, - "filesWritten", filesWritten, - ) if logEnabled { appendManifestLogWarning(cmd, logPath, release.RenderReleasePlanLog(release.ReleasePlanLog{ SchemaVersion: "v1", @@ -536,8 +404,8 @@ func newReleasePlanCmd() *cobra.Command { Targets: opts.Targets, IncludeConnectors: opts.IncludeConnectors, IncludeConnections: opts.IncludeConnections, - AssetsByTarget: assetsByTarget, - PublishByTarget: publishByTarget, + AssetsByTarget: planResult.AssetsByTarget, + PublishByTarget: planResult.PublishByTarget, })) } _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Generated selective plan files for targets: %s\n", strings.Join(opts.Targets, ",")) @@ -666,6 +534,15 @@ func renderTypeCountTable(w io.Writer, title string, counts []release.AssetTypeC }) } +// makeTypeCountRenderer adapts renderTypeCountTable to release.TypeCountRenderer +// so release.BuildPlan can render per-target type-count tables without depending +// on cmd-only presentation helpers. +func makeTypeCountRenderer(w io.Writer) release.TypeCountRenderer { + return func(title string, counts []release.AssetTypeCount, total int) error { + return renderTypeCountTable(w, title, counts, total) + } +} + func renderThemedTable(w io.Writer, rows interface{}, columns []output.Column) error { cfg, _ := loadConfig() // best-effort; nil cfg falls back to defaults f := output.New(output.FormatTable, w, resolveTableStyle(cfg)) diff --git a/internal/release/build.go b/internal/release/build.go new file mode 100644 index 0000000..4c3cde7 --- /dev/null +++ b/internal/release/build.go @@ -0,0 +1,203 @@ +package release + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +// AssetWriter persists assets to disk in a caller-selected output format +// (csv/json/yaml); see WriteAssetsCSV/WriteAssetsJSON/WriteAssetsYAML. +type AssetWriter func(path string, assets []Asset, fields []string) error + +// TypeCountRenderer renders a per-type asset count table (typically to +// stderr) with the given title. Implementations may depend on presentation +// concerns (theming, output format) that live outside this package. +type TypeCountRenderer func(title string, counts []AssetTypeCount, total int) error + +// PlanOptions configures BuildPlan's shared per-target pipeline: resolve -> +// filter missing-transitive -> validate -> write package -> compute publish +// assets -> write publish file, run once per target, followed by an optional +// connectors package file. Full-deployment mode and selective/tag mode differ +// only in how the initial asset set (Assets/ConnectorSourceAssets) is +// resolved and in a handful of mode-specific log messages, captured here via +// the OnTargetProcessing/OnTargetFilesGenerated hooks and CompletedLabel. +type PlanOptions struct { + // Assets is the resolved asset set used as the per-target filter/validate + // source (full mode: seed-resolved assets; selective mode: policy-filtered + // tag assets). + Assets []Asset + // ConnectorSourceAssets is the asset set the optional connectors package + // file is built from. + ConnectorSourceAssets []Asset + + Targets []string + FilterMissingTransitive bool + TargetResolutionOptions TargetResolutionOptions + + OutputRoot string + // PackageFileBaseName is the per-target package file name without its + // extension, e.g. "full_build.package" or "tag_build.package". + PackageFileBaseName string + PlanExt string + WriteAssets AssetWriter + PackageFields []string + PublishFields []string + + IncludeConnectors bool + IncludeConnections bool + + InfoEnabled bool + RenderPackageTotals TypeCountRenderer + RenderPublishTotals TypeCountRenderer + RenderConnectorTotals TypeCountRenderer + OnTargetProcessing func(env string) + OnTargetFilesGenerated func(env, packageFile, publishFile string, publishAssetCount int) + + // CompletedLabel distinguishes the final "completed" log line, e.g. + // "full mode" or "selective mode". Empty skips that log line. + CompletedLabel string +} + +// PlanResult carries the per-target outputs needed for the release plan +// manifest log. +type PlanResult struct { + AssetsByTarget map[string][]ManifestLogAsset + PublishByTarget map[string][]ManifestLogAsset + FilesWritten int + ConnectorsFile string +} + +// BuildPlan runs the shared per-target pipeline (resolve -> filter +// missing-transitive -> validate -> write package -> compute publish assets +// -> write publish file) once per target, for either full-deployment or +// selective/tag mode, differing only in how the initial asset set is +// resolved (opts.Assets/opts.ConnectorSourceAssets) and in the mode-specific +// hooks/labels on PlanOptions. +func BuildPlan(ctx context.Context, opts PlanOptions) (PlanResult, error) { + result := PlanResult{ + AssetsByTarget: make(map[string][]ManifestLogAsset, len(opts.Targets)), + PublishByTarget: make(map[string][]ManifestLogAsset, len(opts.Targets)), + } + + for _, env := range opts.Targets { + if opts.OnTargetProcessing != nil { + opts.OnTargetProcessing(env) + } + + envDir := filepath.Join(opts.OutputRoot, strings.ToLower(env)) + if err := os.MkdirAll(envDir, 0o755); err != nil { + return PlanResult{}, fmt.Errorf("creating env directory: %w", err) + } + + envAssets := opts.Assets + if opts.FilterMissingTransitive { + filtered, err := FilterMissingTransitiveForTarget(ctx, env, opts.Assets, opts.TargetResolutionOptions) + if err != nil { + return PlanResult{}, err + } + envAssets = filtered + slog.Info("release plan: missing-transitive filter applied", + "environment", env, + "before", len(opts.Assets), + "after", len(envAssets), + ) + } + + envPackageAssets, err := AnnotateAssetsWithTargetValidation(ctx, env, envAssets, opts.TargetResolutionOptions) + if err != nil { + return PlanResult{}, err + } + envPackageFields := EnsureCurrentTargetStatusField(opts.PackageFields, env) + publishAssets := PublishAssets(envAssets) + + if opts.InfoEnabled { + if opts.RenderPackageTotals != nil { + if renderErr := opts.RenderPackageTotals( + fmt.Sprintf("release plan: package totals by type for %s", env), + AssetCountsByType(envAssets), + len(envAssets), + ); renderErr != nil { + return PlanResult{}, renderErr + } + } + if opts.RenderPublishTotals != nil { + if renderErr := opts.RenderPublishTotals( + fmt.Sprintf("release plan: publish totals by type for %s", env), + AssetCountsByType(publishAssets), + len(publishAssets), + ); renderErr != nil { + return PlanResult{}, renderErr + } + } + } + + packageFile := filepath.Join(envDir, opts.PackageFileBaseName+"."+opts.PlanExt) + if err := opts.WriteAssets(packageFile, envPackageAssets, envPackageFields); err != nil { + return PlanResult{}, err + } + result.FilesWritten++ + + publishFile := filepath.Join(envDir, "publish_assets."+opts.PlanExt) + if err := opts.WriteAssets(publishFile, publishAssets, opts.PublishFields); err != nil { + return PlanResult{}, err + } + result.FilesWritten++ + + result.AssetsByTarget[env] = manifestLogAssets(envPackageAssets) + result.PublishByTarget[env] = manifestLogAssets(publishAssets) + + if opts.OnTargetFilesGenerated != nil { + opts.OnTargetFilesGenerated(env, packageFile, publishFile, len(publishAssets)) + } + } + + if ShouldWriteConnectorPackage(opts.IncludeConnectors, opts.IncludeConnections) { + connectorAssets := ConnectorPackageAssets(opts.ConnectorSourceAssets) + connectorsFile := filepath.Join(opts.OutputRoot, "connectors.package."+opts.PlanExt) + if err := opts.WriteAssets(connectorsFile, connectorAssets, opts.PackageFields); err != nil { + return PlanResult{}, err + } + result.FilesWritten++ + result.ConnectorsFile = connectorsFile + + if opts.InfoEnabled && opts.RenderConnectorTotals != nil { + if renderErr := opts.RenderConnectorTotals( + "release plan: connector totals by type", + AssetCountsByType(connectorAssets), + len(connectorAssets), + ); renderErr != nil { + return PlanResult{}, renderErr + } + } + slog.Info("release plan: connectors file generated", "connectorsFile", connectorsFile) + } + + if opts.CompletedLabel != "" { + slog.Info("release plan: completed "+opts.CompletedLabel, + "targets", strings.Join(opts.Targets, ","), + "outputRoot", opts.OutputRoot, + "filesWritten", result.FilesWritten, + ) + } + + return result, nil +} + +func manifestLogAssets(assets []Asset) []ManifestLogAsset { + rows := make([]ManifestLogAsset, 0, len(assets)) + for _, a := range assets { + rows = append(rows, ManifestLogAsset{ + ID: a.ID, + Location: a.Location, + Type: a.Type, + Path: a.Path, + Dependency: a.Dependency, + Status: a.Status, + }) + } + return rows +} diff --git a/internal/release/build_test.go b/internal/release/build_test.go new file mode 100644 index 0000000..e920cb2 --- /dev/null +++ b/internal/release/build_test.go @@ -0,0 +1,238 @@ +package release + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jbrazda/iics-cli/internal/client" +) + +// newBuildPlanTestServer returns an httptest server that fakes v3 login and +// lookup responses for BuildPlan's target-resolution calls (FilterMissingTransitiveForTarget, +// AnnotateAssetsWithTargetValidation). An asset is reported as already existing in the +// target when its path contains "Existing"; every other asset is reported missing. +func newBuildPlanTestServer(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) { + resp := client.LoginResponse{ + Products: []client.Product{{Name: "Data Integration", BaseAPIURL: "http://" + r.Host}}, + UserInfo: client.UserInfo{SessionID: "test-session", ID: "u1"}, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + }) + mux.HandleFunc("/public/core/v3/lookup", func(w http.ResponseWriter, r *http.Request) { + var req client.LookupRequest + _ = json.NewDecoder(r.Body).Decode(&req) + var resp client.LookupResponse + for _, o := range req.Objects { + if strings.Contains(o.Path, "Existing") { + resp.Objects = append(resp.Objects, client.LookupResult{ID: "id-" + o.Path, Path: o.Path, Type: o.Type}) + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +// setEnvTargetCredentials points target-profile resolution at the given +// test server via the CI env-var fallback (IICS_USER_/IICS_PWD_/ +// IICS_LOGIN_URL_), isolated from any real ~/.iics/config.yaml via HOME. +func setEnvTargetCredentials(t *testing.T, target, loginURL string) { + t.Helper() + t.Setenv("HOME", t.TempDir()) + key := strings.ToUpper(target) + t.Setenv("IICS_USER_"+key, "user") + t.Setenv("IICS_PWD_"+key, "pass") + t.Setenv("IICS_LOGIN_URL_"+key, loginURL) +} + +func readCSVRows(t *testing.T, path string) [][]string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%s) error = %v", path, err) + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + rows := make([][]string, len(lines)) + for i, l := range lines { + rows[i] = strings.Split(l, ",") + } + return rows +} + +func TestBuildPlanFullMode(t *testing.T) { + srv := newBuildPlanTestServer(t) + setEnvTargetCredentials(t, "QA", srv.URL+"/login") + + assets := []Asset{ + {Location: "Explore/A.PROCESS", Path: "Explore/A", Type: "PROCESS", Dependency: "explicit"}, + {Location: "Explore/Missing.GUIDE", Path: "Explore/Missing", Type: "GUIDE", Dependency: "transitive"}, + {Location: "Explore/Existing.TASKFLOW", Path: "Explore/Existing", Type: "TASKFLOW", Dependency: "transitive"}, + } + + outputRoot := t.TempDir() + var generated []string + result, err := BuildPlan(context.Background(), PlanOptions{ + Assets: assets, + ConnectorSourceAssets: assets, + Targets: []string{"QA"}, + FilterMissingTransitive: true, + OutputRoot: outputRoot, + PackageFileBaseName: "full_build.package", + PlanExt: "csv", + WriteAssets: WriteAssetsCSV, + PackageFields: []string{"location", "type", "path", "dependency"}, + PublishFields: []string{"location", "type", "path", "dependency"}, + OnTargetFilesGenerated: func(env, packageFile, publishFile string, publishAssetCount int) { + generated = append(generated, env, packageFile, publishFile) + if publishAssetCount != 2 { + t.Errorf("publishAssetCount = %d, want 2", publishAssetCount) + } + }, + CompletedLabel: "full mode", + }) + if err != nil { + t.Fatalf("BuildPlan() error = %v", err) + } + + if result.FilesWritten != 2 { + t.Fatalf("FilesWritten = %d, want 2", result.FilesWritten) + } + if result.ConnectorsFile != "" { + t.Fatalf("ConnectorsFile = %q, want empty (no include-connectors requested)", result.ConnectorsFile) + } + if len(generated) != 3 { + t.Fatalf("OnTargetFilesGenerated not invoked as expected: %#v", generated) + } + + // Existing.TASKFLOW is transitive and already exists in QA, so it is + // filtered out; only the explicit PROCESS and the still-missing transitive + // GUIDE remain in the package/publish output. + packageAssets := result.AssetsByTarget["QA"] + if len(packageAssets) != 2 { + t.Fatalf("AssetsByTarget[QA] len = %d, want 2: %#v", len(packageAssets), packageAssets) + } + byLocation := map[string]ManifestLogAsset{} + for _, a := range packageAssets { + byLocation[a.Location] = a + } + if got := byLocation["Explore/A.PROCESS"].Status; got != "missing" { + t.Errorf("Explore/A.PROCESS status = %q, want missing", got) + } + if got := byLocation["Explore/Missing.GUIDE"].Status; got != "missing" { + t.Errorf("Explore/Missing.GUIDE status = %q, want missing", got) + } + if _, ok := byLocation["Explore/Existing.TASKFLOW"]; ok { + t.Fatalf("Explore/Existing.TASKFLOW should have been filtered out: %#v", packageAssets) + } + + publishAssets := result.PublishByTarget["QA"] + if len(publishAssets) != 2 { + t.Fatalf("PublishByTarget[QA] len = %d, want 2: %#v", len(publishAssets), publishAssets) + } + + packagePath := filepath.Join(outputRoot, "qa", "full_build.package.csv") + rows := readCSVRows(t, packagePath) + if len(rows) != 3 { // header + 2 rows + t.Fatalf("package csv rows = %d, want 3: %#v", len(rows), rows) + } + if rows[0][0] != "LOCATION" { + t.Fatalf("unexpected header: %#v", rows[0]) + } + + publishPath := filepath.Join(outputRoot, "qa", "publish_assets.csv") + if _, statErr := os.Stat(publishPath); statErr != nil { + t.Fatalf("expected publish file at %s: %v", publishPath, statErr) + } +} + +func TestBuildPlanSelectiveModeWithConnectors(t *testing.T) { + srv := newBuildPlanTestServer(t) + setEnvTargetCredentials(t, "TST", srv.URL+"/login") + + allFiltered := []Asset{ + {Location: "Explore/A.PROCESS", Path: "Explore/A", Type: "PROCESS", Dependency: "explicit"}, + } + connectorSourceAssets := []Asset{ + {Location: "Explore/A.PROCESS", Path: "Explore/A", Type: "PROCESS", Dependency: "explicit"}, + {Location: "Explore/Conn.AI_CONNECTION", Path: "Explore/Conn", Type: "AI_CONNECTION", Dependency: "explicit"}, + } + + outputRoot := t.TempDir() + var processed []string + result, err := BuildPlan(context.Background(), PlanOptions{ + Assets: allFiltered, + ConnectorSourceAssets: connectorSourceAssets, + Targets: []string{"TST"}, + FilterMissingTransitive: false, + OutputRoot: outputRoot, + PackageFileBaseName: "tag_build.package", + PlanExt: "csv", + WriteAssets: WriteAssetsCSV, + PackageFields: []string{"location", "type", "path", "dependency"}, + PublishFields: []string{"location", "type", "path", "dependency"}, + IncludeConnectors: true, + OnTargetProcessing: func(env string) { + processed = append(processed, env) + }, + CompletedLabel: "selective mode", + }) + if err != nil { + t.Fatalf("BuildPlan() error = %v", err) + } + + if len(processed) != 1 || processed[0] != "TST" { + t.Fatalf("OnTargetProcessing calls = %#v, want [TST]", processed) + } + // 2 files per target + 1 connectors package file. + if result.FilesWritten != 3 { + t.Fatalf("FilesWritten = %d, want 3", result.FilesWritten) + } + wantConnectorsFile := filepath.Join(outputRoot, "connectors.package.csv") + if result.ConnectorsFile != wantConnectorsFile { + t.Fatalf("ConnectorsFile = %q, want %q", result.ConnectorsFile, wantConnectorsFile) + } + + packagePath := filepath.Join(outputRoot, "tst", "tag_build.package.csv") + if _, statErr := os.Stat(packagePath); statErr != nil { + t.Fatalf("expected package file at %s: %v", packagePath, statErr) + } + + connectorRows := readCSVRows(t, wantConnectorsFile) + if len(connectorRows) != 2 { // header + 1 connector asset + t.Fatalf("connector csv rows = %d, want 2: %#v", len(connectorRows), connectorRows) + } + if connectorRows[1][0] != "Explore/Conn.AI_CONNECTION" { + t.Fatalf("unexpected connector row: %#v", connectorRows[1]) + } +} + +func TestBuildPlanPropagatesTargetResolutionError(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + // No IICS_USER_*/IICS_PWD_* env vars and no config profile for "UNKNOWNENV": + // resolveTargetClient must fail, and BuildPlan must surface that error. + _, err := BuildPlan(context.Background(), PlanOptions{ + Assets: []Asset{{Location: "Explore/A.PROCESS", Path: "Explore/A", Type: "PROCESS", Dependency: "explicit"}}, + Targets: []string{"UNKNOWNENV"}, + OutputRoot: t.TempDir(), + PackageFileBaseName: "tag_build.package", + PlanExt: "csv", + WriteAssets: WriteAssetsCSV, + PackageFields: []string{"location"}, + PublishFields: []string{"location"}, + }) + if err == nil { + t.Fatal("expected error for unresolvable target profile") + } +} From 727bdbee5155f76da76d4e3de61a76e86807bcd4 Mon Sep 17 00:00:00 2001 From: Jaroslav Brazda Date: Wed, 16 Sep 2026 13:52:55 -0400 Subject: [PATCH 3/5] fix(output): make wrap-cell measure ANSI-colored words by visible length wrapParagraph compared word length using raw len(), which counts ANSI escape bytes. A short colored status like "found" was measured as longer than the column width and hard-split at a raw byte offset, slicing through the escape sequence (e.g. "foun"/"d" across two physical rows in the release plan dependency status table). Measure and accumulate using visibleLen instead, and only fall back to stripped plain text when a word's visible length genuinely exceeds the column width. --- internal/output/wrap.go | 20 ++++++++++++++++---- internal/output/wrap_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/internal/output/wrap.go b/internal/output/wrap.go index c1d4c3d..55c5268 100644 --- a/internal/output/wrap.go +++ b/internal/output/wrap.go @@ -27,27 +27,39 @@ func wrapParagraph(s string, width int) []string { } var lines []string var cur strings.Builder + curLen := 0 flush := func() { lines = append(lines, cur.String()) cur.Reset() + curLen = 0 } for _, word := range strings.Fields(s) { - for len(word) > width { + wordLen := visibleLen(word) + for wordLen > width { if cur.Len() > 0 { flush() } - lines = append(lines, word[:width]) - word = word[width:] + // A word wider than the column can't be split without cutting + // through its ANSI escape bytes, so fall back to its plain text + // for the overflow line (styling is dropped, same trade-off + // TruncateCell makes on a truncated line). + r := []rune(stripANSIText(word)) + lines = append(lines, string(r[:width])) + word = string(r[width:]) + wordLen = len(r) - width } switch { case cur.Len() == 0: cur.WriteString(word) - case cur.Len()+1+len(word) <= width: + curLen = wordLen + case curLen+1+wordLen <= width: cur.WriteByte(' ') cur.WriteString(word) + curLen += 1 + wordLen default: flush() cur.WriteString(word) + curLen = wordLen } } if cur.Len() > 0 || len(lines) == 0 { diff --git a/internal/output/wrap_test.go b/internal/output/wrap_test.go index 872934e..2519a96 100644 --- a/internal/output/wrap_test.go +++ b/internal/output/wrap_test.go @@ -25,6 +25,32 @@ func TestWrapCell(t *testing.T) { } } +func TestWrapCellANSI(t *testing.T) { + // An ANSI-colored word whose visible length fits the column must not be + // split mid-escape-sequence just because its raw byte length exceeds + // width (regression: "found" rendered as "\x1b[1;32mfound\x1b[0m" was + // hard-split into "foun" / "d"). + colored := "\x1b[1;32mfound\x1b[0m" + if got := WrapCell(colored, 12); got != colored { + t.Errorf("WrapCell corrupted a short colored word: got %q, want %q", got, colored) + } + if visibleLen(colored) != len("found") { + t.Fatalf("test fixture invariant broken: visibleLen(%q) = %d", colored, visibleLen(colored)) + } + + // A colored word whose visible length genuinely exceeds width still + // hard-splits, without panicking or leaving stray escape bytes; styling + // on the overflow line is dropped, matching TruncateCell's behavior. + longColored := "\x1b[1;32mexceedinglylongstatus\x1b[0m" + got := WrapCell(longColored, 8) + if strings.Contains(got, "\x1b") { + t.Errorf("hard-split of colored word left stray escape bytes: %q", got) + } + if !strings.Contains(got, "exceedin") { + t.Errorf("hard-split of colored word = %q", got) + } +} + func TestTruncateCell(t *testing.T) { if got := TruncateCell("hello world", 8); got != "hello w…" { t.Errorf("TruncateCell = %q", got) From 95b8a764a287cf96722c279473433f9d4aae6296 Mon Sep 17 00:00:00 2001 From: Jaroslav Brazda Date: Wed, 16 Sep 2026 13:53:04 -0400 Subject: [PATCH 4/5] perf(release): batch target existence checks and reuse validation matrix `release plan` was validating asset existence per target with one Lookup/GetConnectionByName API call per asset, and recomputing that same existence matrix twice whenever --verbose was on: once to render the dependency status table, again inside BuildPlan to annotate the package/publish files. - validateAssetsExistInTarget batches all non-Connection assets for a target into a single Lookup call instead of one call per asset, matching the batching pattern already used in internal/dependencies. - ValidateAssetsForTargets computes the per-target validation matrix once; cmd/release.go threads it through PlanOptions.Validations so BuildPlan's missing-transitive filter and package annotation reuse it instead of re-querying the target. For a 65-asset / 2-target selective plan with --verbose, this cuts the API round trips from ~260 down to 2. --- cmd/release.go | 29 ++++--- internal/release/build.go | 33 ++++++-- internal/release/build_test.go | 79 ++++++++++++++++++- internal/release/plan.go | 140 +++++++++++++++++++++++++-------- 4 files changed, 225 insertions(+), 56 deletions(-) diff --git a/cmd/release.go b/cmd/release.go index eab1f78..c2337e3 100644 --- a/cmd/release.go +++ b/cmd/release.go @@ -263,14 +263,17 @@ func newReleasePlanCmd() *cobra.Command { "explicitAssets", stats.ExplicitAssets, "transitiveAssets", stats.TransitiveAssets, ) + validationsByTarget, valErr := release.ValidateAssetsForTargets(context.Background(), opts.Targets, fullAssets, targetResolutionOpts) + if valErr != nil { + return valErr + } if infoEnabled { slog.Info("release plan: full mode dependency status table") if renderErr := renderDependencyStatusTable( - context.Background(), logWriter, fullAssets, opts.Targets, - targetResolutionOpts, + validationsByTarget, ); renderErr != nil { return renderErr } @@ -285,6 +288,7 @@ func newReleasePlanCmd() *cobra.Command { Targets: opts.Targets, FilterMissingTransitive: filterMissingTrans, TargetResolutionOptions: targetResolutionOpts, + Validations: validationsByTarget, OutputRoot: outputRoot, PackageFileBaseName: "full_build.package", PlanExt: planExt, @@ -344,14 +348,17 @@ func newReleasePlanCmd() *cobra.Command { } allFiltered := release.ApplyPolicies(assets, opts.IncludeConnectors, opts.IncludeConnections, excludes) + validationsByTarget, valErr := release.ValidateAssetsForTargets(context.Background(), opts.Targets, allFiltered, targetResolutionOpts) + if valErr != nil { + return valErr + } if infoEnabled { slog.Info("release plan: dependency status table") if err := renderDependencyStatusTable( - context.Background(), logWriter, allFiltered, opts.Targets, - targetResolutionOpts, + validationsByTarget, ); err != nil { return err } @@ -367,6 +374,7 @@ func newReleasePlanCmd() *cobra.Command { Targets: opts.Targets, FilterMissingTransitive: filterMissingTrans, TargetResolutionOptions: targetResolutionOpts, + Validations: validationsByTarget, OutputRoot: outputRoot, PackageFileBaseName: "tag_build.package", PlanExt: planExt, @@ -469,11 +477,10 @@ func resolvePlanAssetsWriter(raw string) (func(string, []release.Asset, []string } func renderDependencyStatusTable( - ctx context.Context, w io.Writer, assets []release.Asset, targets []string, - opts release.TargetResolutionOptions, + validationsByTarget map[string]map[string]release.AssetValidation, ) error { sortedAssets := release.SortAssetsByLocation(assets) tableRows := make([]map[string]interface{}, len(sortedAssets)) @@ -489,14 +496,12 @@ func renderDependencyStatusTable( } for _, target := range targets { - validations, err := release.ValidateAssetsForTarget(ctx, target, sortedAssets, opts) - if err != nil { - return fmt.Errorf("profile %q: %w", target, err) - } + validations := validationsByTarget[target] key := strings.ReplaceAll(target, "-", "_") for i := range tableRows { - tableRows[i]["status_"+key] = validations[i].Status - tableRows[i]["warning_"+key] = validations[i].Warning + v := validations[sortedAssets[i].Location] + tableRows[i]["status_"+key] = v.Status + tableRows[i]["warning_"+key] = v.Warning } } diff --git a/internal/release/build.go b/internal/release/build.go index 4c3cde7..5d41fc5 100644 --- a/internal/release/build.go +++ b/internal/release/build.go @@ -38,6 +38,13 @@ type PlanOptions struct { FilterMissingTransitive bool TargetResolutionOptions TargetResolutionOptions + // Validations, when set, is a precomputed target -> Location -> + // AssetValidation matrix (e.g. from ValidateAssetsForTargets, already + // used to render a verbose dependency status table) reused here to skip + // re-querying the target for assets it was already validated against. + // A target missing from the map falls back to querying live. + Validations map[string]map[string]AssetValidation + OutputRoot string // PackageFileBaseName is the per-target package file name without its // extension, e.g. "full_build.package" or "tag_build.package". @@ -93,13 +100,19 @@ func BuildPlan(ctx context.Context, opts PlanOptions) (PlanResult, error) { return PlanResult{}, fmt.Errorf("creating env directory: %w", err) } + validations := opts.Validations[env] + envAssets := opts.Assets if opts.FilterMissingTransitive { - filtered, err := FilterMissingTransitiveForTarget(ctx, env, opts.Assets, opts.TargetResolutionOptions) - if err != nil { - return PlanResult{}, err + if validations != nil { + envAssets = FilterMissingTransitiveWithValidations(opts.Assets, validations) + } else { + filtered, err := FilterMissingTransitiveForTarget(ctx, env, opts.Assets, opts.TargetResolutionOptions) + if err != nil { + return PlanResult{}, err + } + envAssets = filtered } - envAssets = filtered slog.Info("release plan: missing-transitive filter applied", "environment", env, "before", len(opts.Assets), @@ -107,9 +120,15 @@ func BuildPlan(ctx context.Context, opts PlanOptions) (PlanResult, error) { ) } - envPackageAssets, err := AnnotateAssetsWithTargetValidation(ctx, env, envAssets, opts.TargetResolutionOptions) - if err != nil { - return PlanResult{}, err + var envPackageAssets []Asset + if validations != nil { + envPackageAssets = AnnotateAssetsWithValidations(envAssets, validations) + } else { + annotated, err := AnnotateAssetsWithTargetValidation(ctx, env, envAssets, opts.TargetResolutionOptions) + if err != nil { + return PlanResult{}, err + } + envPackageAssets = annotated } envPackageFields := EnsureCurrentTargetStatusField(opts.PackageFields, env) publishAssets := PublishAssets(envAssets) diff --git a/internal/release/build_test.go b/internal/release/build_test.go index e920cb2..a21a948 100644 --- a/internal/release/build_test.go +++ b/internal/release/build_test.go @@ -17,8 +17,11 @@ import ( // lookup responses for BuildPlan's target-resolution calls (FilterMissingTransitiveForTarget, // AnnotateAssetsWithTargetValidation). An asset is reported as already existing in the // target when its path contains "Existing"; every other asset is reported missing. -func newBuildPlanTestServer(t *testing.T) *httptest.Server { +// The returned counter tracks how many /lookup requests were made, so tests +// can assert assets are validated in as few batched calls as possible. +func newBuildPlanTestServer(t *testing.T) (*httptest.Server, *int) { t.Helper() + lookupCalls := 0 mux := http.NewServeMux() mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) { resp := client.LoginResponse{ @@ -29,6 +32,7 @@ func newBuildPlanTestServer(t *testing.T) *httptest.Server { _ = json.NewEncoder(w).Encode(resp) }) mux.HandleFunc("/public/core/v3/lookup", func(w http.ResponseWriter, r *http.Request) { + lookupCalls++ var req client.LookupRequest _ = json.NewDecoder(r.Body).Decode(&req) var resp client.LookupResponse @@ -42,7 +46,7 @@ func newBuildPlanTestServer(t *testing.T) *httptest.Server { }) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) - return srv + return srv, &lookupCalls } // setEnvTargetCredentials points target-profile resolution at the given @@ -72,7 +76,7 @@ func readCSVRows(t *testing.T, path string) [][]string { } func TestBuildPlanFullMode(t *testing.T) { - srv := newBuildPlanTestServer(t) + srv, lookupCalls := newBuildPlanTestServer(t) setEnvTargetCredentials(t, "QA", srv.URL+"/login") assets := []Asset{ @@ -155,10 +159,17 @@ func TestBuildPlanFullMode(t *testing.T) { if _, statErr := os.Stat(publishPath); statErr != nil { t.Fatalf("expected publish file at %s: %v", publishPath, statErr) } + + // The missing-transitive filter and the package annotation each validate + // existence for the same 3 assets; both must batch all their assets into + // a single /lookup call rather than one call per asset. + if *lookupCalls != 2 { + t.Errorf("lookupCalls = %d, want 2 (one batched call per validation pass)", *lookupCalls) + } } func TestBuildPlanSelectiveModeWithConnectors(t *testing.T) { - srv := newBuildPlanTestServer(t) + srv, _ := newBuildPlanTestServer(t) setEnvTargetCredentials(t, "TST", srv.URL+"/login") allFiltered := []Asset{ @@ -218,6 +229,66 @@ func TestBuildPlanSelectiveModeWithConnectors(t *testing.T) { } } +func TestBuildPlanReusesPrecomputedValidations(t *testing.T) { + // No profile or CI env credentials configured for "QA" at all: if + // BuildPlan resolved a target client despite a precomputed validation + // matrix being supplied, resolveTargetClient would fail immediately. + // Succeeding here proves the missing-transitive filter and package + // annotation both reused the matrix instead of validating over the + // network. + t.Setenv("HOME", t.TempDir()) + + assets := []Asset{ + {Location: "Explore/A.PROCESS", Path: "Explore/A", Type: "PROCESS", Dependency: "explicit"}, + {Location: "Explore/Missing.GUIDE", Path: "Explore/Missing", Type: "GUIDE", Dependency: "transitive"}, + {Location: "Explore/Existing.TASKFLOW", Path: "Explore/Existing", Type: "TASKFLOW", Dependency: "transitive"}, + } + validations := map[string]map[string]AssetValidation{ + "QA": { + "Explore/A.PROCESS": {Status: "missing"}, + "Explore/Missing.GUIDE": {Status: "missing"}, + "Explore/Existing.TASKFLOW": {Status: "found"}, + }, + } + + outputRoot := t.TempDir() + result, err := BuildPlan(context.Background(), PlanOptions{ + Assets: assets, + ConnectorSourceAssets: assets, + Targets: []string{"QA"}, + FilterMissingTransitive: true, + Validations: validations, + OutputRoot: outputRoot, + PackageFileBaseName: "full_build.package", + PlanExt: "csv", + WriteAssets: WriteAssetsCSV, + PackageFields: []string{"location", "type", "path", "dependency"}, + PublishFields: []string{"location", "type", "path", "dependency"}, + CompletedLabel: "full mode", + }) + if err != nil { + t.Fatalf("BuildPlan() error = %v", err) + } + + // Existing.TASKFLOW is transitive and marked "found" in the precomputed + // matrix, so it is filtered out; the explicit PROCESS and the still- + // missing transitive GUIDE remain, stamped with the matrix's statuses. + packageAssets := result.AssetsByTarget["QA"] + if len(packageAssets) != 2 { + t.Fatalf("AssetsByTarget[QA] len = %d, want 2: %#v", len(packageAssets), packageAssets) + } + byLocation := map[string]ManifestLogAsset{} + for _, a := range packageAssets { + byLocation[a.Location] = a + } + if got := byLocation["Explore/A.PROCESS"].Status; got != "missing" { + t.Errorf("Explore/A.PROCESS status = %q, want missing", got) + } + if _, ok := byLocation["Explore/Existing.TASKFLOW"]; ok { + t.Fatalf("Explore/Existing.TASKFLOW should have been filtered out via precomputed validations: %#v", packageAssets) + } +} + func TestBuildPlanPropagatesTargetResolutionError(t *testing.T) { t.Setenv("HOME", t.TempDir()) // No IICS_USER_*/IICS_PWD_* env vars and no config profile for "UNKNOWNENV": diff --git a/internal/release/plan.go b/internal/release/plan.go index 5b63e17..707d993 100644 --- a/internal/release/plan.go +++ b/internal/release/plan.go @@ -270,20 +270,24 @@ func FilterMissingTransitiveForTarget(ctx context.Context, targetProfileName str if err != nil { return nil, err } + validations := validateAssetsExistInTarget(ctx, tc, assets) + return FilterMissingTransitiveWithValidations(assets, validations), nil +} +// FilterMissingTransitiveWithValidations applies the missing-transitive policy +// using a precomputed validation matrix (asset Location -> AssetValidation), +// avoiding a network round trip when the matrix was already computed +// elsewhere (e.g. by ValidateAssetsForTargets for the status table). +func FilterMissingTransitiveWithValidations(assets []Asset, validations map[string]AssetValidation) []Asset { missingByLocation := make(map[string]bool, len(assets)) for _, a := range assets { if a.Dependency != "transitive" { continue } - exists, existsErr := assetExistsInTarget(ctx, tc, a) - if existsErr != nil { - return nil, fmt.Errorf("checking target %q for %s: %w", targetProfileName, a.Location, existsErr) - } - missingByLocation[a.Location] = !exists + v, ok := validations[a.Location] + missingByLocation[a.Location] = !ok || v.Status != "found" } - - return ApplyMissingTransitivePolicy(assets, missingByLocation), nil + return ApplyMissingTransitivePolicy(assets, missingByLocation) } func ValidateAssetsForTarget(ctx context.Context, targetProfileName string, assets []Asset, opts TargetResolutionOptions) ([]AssetValidation, error) { @@ -291,19 +295,32 @@ func ValidateAssetsForTarget(ctx context.Context, targetProfileName string, asse if err != nil { return nil, err } - + validations := validateAssetsExistInTarget(ctx, tc, assets) out := make([]AssetValidation, len(assets)) for i, a := range assets { - exists, existsErr := assetExistsInTarget(ctx, tc, a) - if existsErr != nil { - out[i] = AssetValidation{Status: "unknown", Warning: existsErr.Error()} - continue + if v, ok := validations[a.Location]; ok { + out[i] = v + } else { + out[i] = AssetValidation{Status: "unknown"} } - if exists { - out[i] = AssetValidation{Status: "found"} - continue + } + return out, nil +} + +// ValidateAssetsForTargets computes an existence validation matrix for every +// (target, asset) pair, batching each target's non-Connection assets into a +// single Lookup call rather than one call per asset. The result (keyed by +// target, then by asset Location) can be reused across the status table +// render and BuildPlan's package/publish annotation instead of re-querying +// the same assets twice. +func ValidateAssetsForTargets(ctx context.Context, targets []string, assets []Asset, opts TargetResolutionOptions) (map[string]map[string]AssetValidation, error) { + out := make(map[string]map[string]AssetValidation, len(targets)) + for _, target := range targets { + tc, _, err := resolveTargetClient(target, opts) + if err != nil { + return nil, fmt.Errorf("profile %q: %w", target, err) } - out[i] = AssetValidation{Status: "missing"} + out[target] = validateAssetsExistInTarget(ctx, tc, assets) } return out, nil } @@ -326,6 +343,21 @@ func AnnotateAssetsWithTargetValidation(ctx context.Context, targetProfileName s return out, nil } +// AnnotateAssetsWithValidations returns a copy of assets with Status/Warning +// populated from a precomputed validation matrix (asset Location -> +// AssetValidation), avoiding a network round trip. +func AnnotateAssetsWithValidations(assets []Asset, validations map[string]AssetValidation) []Asset { + out := make([]Asset, len(assets)) + copy(out, assets) + for i := range out { + if v, ok := validations[out[i].Location]; ok { + out[i].Status = v.Status + out[i].Warning = v.Warning + } + } + return out +} + func statusFieldForTarget(target string) string { return fmt.Sprintf("status (%s)", strings.ToLower(strings.TrimSpace(target))) } @@ -564,32 +596,74 @@ func resolveTargetClient(targetProfileName string, opts TargetResolutionOptions) return client.NewClient(loginURL, profile.Username, profile.Password, clientOptions...), resolvedProfileName, nil } -func assetExistsInTarget(ctx context.Context, tc *client.Client, a Asset) (bool, error) { - if a.Type == "Connection" { - name := a.Path - if idx := strings.LastIndex(name, "/"); idx >= 0 { - name = name[idx+1:] - } - _, err := tc.GetConnectionByName(ctx, name) - if err == nil { - return true, nil - } - if isMissingConnectionError(err) { - return false, nil +// validateAssetsExistInTarget checks existence of every asset against tc, +// batching everything except Connection-typed assets (which have no bulk +// lookup API) into a single Lookup call instead of one round trip per asset. +// A lookup failure is recorded as an "unknown" status against the affected +// assets rather than aborting the whole pass, matching the previous +// per-asset error handling. +func validateAssetsExistInTarget(ctx context.Context, tc *client.Client, assets []Asset) map[string]AssetValidation { + out := make(map[string]AssetValidation, len(assets)) + + var lookupAssets []Asset + var lookupObjs []client.LookupObject + for _, a := range assets { + if a.Type == "Connection" { + out[a.Location] = validateConnectionExists(ctx, tc, a) + continue } - return false, err + lookupAssets = append(lookupAssets, a) + lookupObjs = append(lookupObjs, client.LookupObject{Path: a.Path, Type: a.Type}) } - resp, err := tc.Lookup(ctx, []client.LookupObject{{Path: a.Path, Type: a.Type}}) + if len(lookupObjs) == 0 { + return out + } + + resp, err := tc.Lookup(ctx, lookupObjs) if err != nil { var apiErr *client.APIError if errors.As(err, &apiErr) && bytes.Contains(apiErr.ResponseBody, []byte(`"V3API_LookupError_012"`)) { - return false, nil + for _, a := range lookupAssets { + out[a.Location] = AssetValidation{Status: "missing"} + } + return out } - return false, err + warning := err.Error() + for _, a := range lookupAssets { + out[a.Location] = AssetValidation{Status: "unknown", Warning: warning} + } + return out + } + + found := make(map[string]bool, len(resp.Objects)) + for _, obj := range resp.Objects { + found[client.BuildLookupMatchKey(obj.Path, obj.Type)] = true + } + for _, a := range lookupAssets { + if found[client.BuildLookupMatchKey(a.Path, a.Type)] { + out[a.Location] = AssetValidation{Status: "found"} + } else { + out[a.Location] = AssetValidation{Status: "missing"} + } + } + return out +} + +func validateConnectionExists(ctx context.Context, tc *client.Client, a Asset) AssetValidation { + name := a.Path + if idx := strings.LastIndex(name, "/"); idx >= 0 { + name = name[idx+1:] + } + _, err := tc.GetConnectionByName(ctx, name) + if err == nil { + return AssetValidation{Status: "found"} + } + if isMissingConnectionError(err) { + return AssetValidation{Status: "missing"} } - return len(resp.Objects) > 0, nil + return AssetValidation{Status: "unknown", Warning: err.Error()} } func isMissingConnectionError(err error) bool { From 6417fbc834ebcb4dbe737737690040c85c35ad0f Mon Sep 17 00:00:00 2001 From: Jaroslav Brazda Date: Wed, 16 Sep 2026 16:07:58 -0400 Subject: [PATCH 5/5] feat(output): default non-TTY table output to markdown instead of plain Piped/redirected table output with no explicitly configured theme now renders as markdown (colorless GFM table, already safe for scripting) rather than plain ASCII borders. An explicitly configured theme other than markdown/gh still downgrades to plain when non-interactive, same as before. resolveTableStyle previously seeded style.Theme with the literal string "default" as its unset sentinel, which made effectiveTheme's unset-theme branch unreachable from the main CLI flow. It now leaves Theme empty so effectiveTheme can distinguish "nothing configured" from an explicit --theme default. --- README.md | 12 +++++++----- cmd/root.go | 9 +++++++-- docs/CLAUDE.md | 4 +++- internal/output/table.go | 15 ++++++++++++--- internal/output/table_test.go | 31 +++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3d568aa..59d8879 100644 --- a/README.md +++ b/README.md @@ -145,14 +145,16 @@ The `style` section controls table output appearance. Available themes: | `default` | Unicode rounded borders, cyan bold headers (TTY only) | | `minimal` | No borders, colored bold headers with unicode underline | | `compact` | No borders, gray bold headers, 1-space column gap (TTY only) | -| `plain` | ASCII borders, no color - used automatically for non-TTY output | +| `plain` | ASCII borders, no color - used automatically when a non-default theme is set but output is non-TTY | | `markdown` | GitHub-flavored markdown table, no color, always rendered regardless of TTY | | `gh` | GitHub CLI-style: no borders, no separator, plain headers, no color, always rendered regardless of TTY | -Non-TTY output (piped, redirected) always uses `plain` regardless of the configured theme. -The `markdown` and `gh` themes are exceptions - they always render as-is even when output -is piped, since they are already colorless. -The `NO_COLOR` environment variable is also respected. +Non-TTY output (piped, redirected) with no theme configured uses `markdown` by default, since it +is already colorless and safe to read or parse without a terminal. If you explicitly configure a +different theme (`--theme compact`, `style.theme: minimal`, etc.), non-TTY output falls back to +`plain` instead of forcing that theme's formatting. The `markdown` and `gh` themes are exceptions - +they always render as-is even when output is piped, since they are already colorless. +The `NO_COLOR` environment variable is also respected and always forces `plain`. The `style.headerColor` field accepts a lipgloss color string (`"6"` for cyan, `"244"` for gray, `"#FF0000"` for hex) and overrides the built-in header color for `default` and `minimal` diff --git a/cmd/root.go b/cmd/root.go index 738f0d5..72d701d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -374,9 +374,14 @@ func getClient(cmd *cobra.Command) (*client.Client, error) { // Precedence (highest first): // // --no-color flag / NO_COLOR env > --theme flag > IICS_THEME env > -// config style.noColor > config style.theme > "default" +// config style.noColor > config style.theme > unset (effectiveTheme's default) +// +// Theme is left unset ("") rather than defaulting to "default" here, so +// effectiveTheme can tell "nothing configured" apart from an explicit +// "--theme default" and apply its own TTY-aware default (default on a +// terminal, markdown when piped/redirected). func resolveTableStyle(cfg *config.Config) output.TableStyle { - style := output.TableStyle{Theme: "default"} + style := output.TableStyle{} // Config-level preferences (lowest named precedence) if cfg != nil { diff --git a/docs/CLAUDE.md b/docs/CLAUDE.md index 946ecc4..c9e21f1 100644 --- a/docs/CLAUDE.md +++ b/docs/CLAUDE.md @@ -128,7 +128,9 @@ _ = f.Format(data, cols) ``` `TableStyle.NoColor = true` forces plain ASCII rendering (no color, no Unicode borders). -When the output writer is not a TTY the renderer automatically falls back to plain style. +When the output writer is not a TTY and no theme is explicitly configured, the renderer +defaults to `markdown` (colorless, pipe/script-friendly) instead of `default`; an explicitly +configured theme other than `markdown`/`gh` still falls back to `plain` on a non-TTY writer. --- diff --git a/internal/output/table.go b/internal/output/table.go index d5d1b4f..7a9a91d 100644 --- a/internal/output/table.go +++ b/internal/output/table.go @@ -116,14 +116,23 @@ func adaptWidth(w io.Writer, style TableStyle, theme string) int { return 0 } -// effectiveTheme resolves the theme to use, downgrading to "plain" when -// no-color is set or the output writer is not a TTY. +// effectiveTheme resolves the theme to use. No-color always downgrades to +// "plain". On a non-TTY writer, an unset theme now defaults to "markdown" +// (script/pipe-friendly, and already colorless) rather than "plain"; an +// explicitly-configured theme other than markdown/gh still downgrades to +// "plain" there, same as before. // The "markdown" and "gh" themes are always colorless and bypass the TTY check. func effectiveTheme(w io.Writer, style TableStyle) string { if style.Theme == "markdown" || style.Theme == "gh" { return style.Theme } - if style.NoColor || !isTerminal(w) { + if style.NoColor { + return "plain" + } + if !isTerminal(w) { + if style.Theme == "" { + return "markdown" + } return "plain" } if style.Theme == "" { diff --git a/internal/output/table_test.go b/internal/output/table_test.go index 4f92637..fd5bb24 100644 --- a/internal/output/table_test.go +++ b/internal/output/table_test.go @@ -93,6 +93,37 @@ func TestGHThemeNonTTY(t *testing.T) { } } +// TestEffectiveThemeDefaultNonTTY verifies that an unset theme resolves to +// "markdown" (not "plain") on a non-TTY writer, and "default" on a TTY. +func TestEffectiveThemeDefaultNonTTY(t *testing.T) { + var buf bytes.Buffer + if got := effectiveTheme(&buf, TableStyle{}); got != "markdown" { + t.Errorf("effectiveTheme() with unset theme on non-TTY = %q, want %q", got, "markdown") + } +} + +// TestEffectiveThemeExplicitNonTTYStillDowngrades verifies that an +// explicitly-configured non-colorless theme (anything but markdown/gh) still +// downgrades to "plain" on a non-TTY writer, unaffected by the new +// unset-theme default. +func TestEffectiveThemeExplicitNonTTYStillDowngrades(t *testing.T) { + var buf bytes.Buffer + for _, theme := range []string{"default", "compact", "minimal", "plain"} { + if got := effectiveTheme(&buf, TableStyle{Theme: theme}); got != "plain" { + t.Errorf("effectiveTheme() with Theme:%q on non-TTY = %q, want %q", theme, got, "plain") + } + } +} + +// TestEffectiveThemeNoColorOverridesMarkdownDefault verifies that NoColor +// still forces "plain" on a non-TTY writer even when no theme is set. +func TestEffectiveThemeNoColorOverridesMarkdownDefault(t *testing.T) { + var buf bytes.Buffer + if got := effectiveTheme(&buf, TableStyle{NoColor: true}); got != "plain" { + t.Errorf("effectiveTheme() with NoColor on non-TTY = %q, want %q", got, "plain") + } +} + // TestRowFooterPlural verifies that 3 rows appends "3 rows" on a separate line. func TestRowFooterPlural(t *testing.T) { var buf bytes.Buffer