gen-manifest: optimize checksum calculation

cmd/gen-manifests/checksums.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/osbuild/image-builder/pkg/container"
+	"github.com/osbuild/image-builder/pkg/depsolvednf"
+	"github.com/osbuild/image-builder/pkg/flatpak"
+	"github.com/osbuild/image-builder/pkg/manifest"
+	"github.com/osbuild/image-builder/pkg/ostree"
+)
+
+// manifestChecksum is the method reponsible for doing the hard work
+func manifestChecksum(b []byte) string {
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])
+}
+
+// Checksums records manifest digests and writes them under dir.
+type Checksums struct {
+	dir       string
+	processed sync.Map // key: checksum basename (no .json), value: hex digest
+}
+
+func newChecksums(dir string) *Checksums {
+	return &Checksums{dir: dir}
+}
+
+func checksumBasename(filename string) string {
+	return strings.TrimSuffix(filepath.Base(filename), ".json")
+}
+
+func writeChecksumFileIfChanged(path, digest string) error {
+	want := digest + "\n"
+	b, err := os.ReadFile(path)
+	if err == nil && strings.TrimSuffix(string(b), "\n") == digest {
+		return nil
+	}
+	return os.WriteFile(path, []byte(want), 0o644) //nolint:gosec // checksum artifacts are world-readable in git
+}
+
+func (c *Checksums) recordManifestChecksum(ms manifest.OSBuildManifest, depsolved map[string]depsolvednf.DepsolveResult, containers map[string][]container.Spec, commits map[string][]ostree.CommitSpec, flatpaks map[string][]flatpak.Spec, cr buildRequest, filename string, metadata bool) error {
+	var buf bytes.Buffer
+	if err := save(&buf, false, ms, depsolved, containers, commits, flatpaks, cr, filename, metadata); err != nil {
+		return err
+	}
+	name := checksumBasename(filename)
+	digest := manifestChecksum(buf.Bytes())
+	path := filepath.Join(c.dir, name)
+	if err := writeChecksumFileIfChanged(path, digest); err != nil {
+		return fmt.Errorf("failed to write checksum %q: %w", path, err)
+	}
+	c.processed.Store(name, digest)
+	return nil
+}
+
+func (c *Checksums) deleteStaleChecksums() error {
+	entries, err := os.ReadDir(c.dir)
+	if err != nil {
+		return fmt.Errorf("failed to read checksum directory %q: %w", c.dir, err)
+	}
+	for _, e := range entries {
+		name := e.Name()
+		if _, ok := c.processed.Load(name); ok || e.IsDir() {
+			continue
+		}
+		if err := os.Remove(filepath.Join(c.dir, name)); err != nil {
+			return fmt.Errorf("failed to remove stale checksum %q: %w", name, err)
+		}
+	}
+	return nil
+}

cmd/gen-manifests/main.go

@@ -11,8 +11,10 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"runtime/debug"
 	"slices"
 	"strings"
@@ -237,6 +239,7 @@ func makeManifestJob(
 	tmpdirRoot string,
 	bootcRemote bool,
 	bootcInstallerRef string,
+	cs *Checksums,
 ) manifestJob {
 	name := bc.Name
 	distroName := distribution.Name()
@@ -384,13 +387,23 @@ func makeManifestJob(
 			Repositories: allRepos,
 			Config:       bc,
 		}
-		err = save(mf, depsolvedSets, containerSpecs, commitSpecs, flatpakSpecs, request, path, filename, metadata)
+		if cs != nil {
+			err = cs.recordManifestChecksum(mf, depsolvedSets, containerSpecs, commitSpecs, flatpakSpecs, request, filename, metadata)
+		} else {
+			fpath := filepath.Join(path, filename)
+			fp, createErr := os.Create(fpath)
+			if createErr != nil {
+				return fmt.Errorf("failed to create output file %q: %s\n", fpath, createErr.Error())
+			}
+			defer fp.Close()
+			err = save(fp, true, mf, depsolvedSets, containerSpecs, commitSpecs, flatpakSpecs, request, filename, metadata)
+		}
 		return
 	}
 	return job
 }
 
-func save(ms manifest.OSBuildManifest, depsolved map[string]depsolvednf.DepsolveResult, containers map[string][]container.Spec, commits map[string][]ostree.CommitSpec, flatpaks map[string][]flatpak.Spec, cr buildRequest, path, filename string, metadata bool) error {
+func save(w io.Writer, indent bool, ms manifest.OSBuildManifest, depsolved map[string]depsolvednf.DepsolveResult, containers map[string][]container.Spec, commits map[string][]ostree.CommitSpec, flatpaks map[string][]flatpak.Spec, cr buildRequest, filename string, metadata bool) error {
 	var data any
 	if metadata {
 		rpmmds := make(map[string]rpmmd.PackageList)
@@ -411,19 +424,19 @@ func save(ms manifest.OSBuildManifest, depsolved map[string]depsolvednf.Depsolve
 	} else {
 		data = ms
 	}
-	b, err := json.MarshalIndent(data, "", "  ")
+	var b []byte
+	var err error
+	if indent {
+		b, err = json.MarshalIndent(data, "", "  ")
+	} else {
+		b, err = json.Marshal(data)
+	}
 	if err != nil {
 		return fmt.Errorf("failed to marshal data for %q: %s\n", filename, err.Error())
 	}
 	b = append(b, '\n') // add new line at end of file
-	fpath := filepath.Join(path, filename)
-	fp, err := os.Create(fpath)
-	if err != nil {
-		return fmt.Errorf("failed to create output file %q: %s\n", fpath, err.Error())
-	}
-	defer fp.Close()
-	if _, err := fp.Write(b); err != nil {
-		return fmt.Errorf("failed to write output file %q: %s\n", fpath, err.Error())
+	if _, err := w.Write(b); err != nil {
+		return fmt.Errorf("failed to write output for %q: %s\n", filename, err.Error())
 	}
 	return nil
 }
@@ -438,7 +451,7 @@ func main() {
 	var nWorkers int
 	var metadata, skipNoconfig, skipNorepos, buildconfigAllowUnknown bool
 	flag.StringVar(&outputDir, "output", "test/data/manifests/", "manifest store directory")
-	flag.IntVar(&nWorkers, "workers", 16, "number of workers to run concurrently")
+	flag.IntVar(&nWorkers, "workers", runtime.NumCPU()+1, "number of workers to run concurrently")
 	flag.StringVar(&cacheRoot, "cache", "/tmp/rpmmd", "rpm metadata cache directory")
 	flag.BoolVar(&metadata, "metadata", true, "store metadata in the file")
 	flag.StringVar(&configPath, "config", "", "image config file to use for all images (overrides -config-list)")
@@ -472,6 +485,9 @@ func main() {
 	var dryRun bool
 	flag.BoolVar(&dryRun, "dry-run", false, "print what manifests would be generated")
 
+	var checksumsOnly bool
+	flag.BoolVar(&checksumsOnly, "checksums-only", false, "write manifest SHA-256 checksum files instead of JSON manifests")
+
 	flag.Parse()
 
 	testedRepoRegistry, err := testrepos.New()
@@ -502,6 +518,11 @@ func main() {
 		panic(fmt.Sprintf("failed to create target directory: %s", err.Error()))
 	}
 
+	var cs *Checksums
+	if checksumsOnly {
+		cs = newChecksums(outputDir)
+	}
+
 	// temporary directory for mocking file embeds with URIs (and anything else
 	// we might need to write temporarily)
 	// We can't use os.MkdirTemp to get an uniquw tmp dir here because the path
@@ -575,7 +596,7 @@ func main() {
 					if dryRun {
 						fmt.Printf("%s,%s,%s,%s\n", distribution.Name(), archName, imgType.Name(), itConfig.Name)
 					} else {
-						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot, false, "")
+						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot, false, "", cs)
 						jobs = append(jobs, job)
 					}
 				}
@@ -642,7 +663,7 @@ func main() {
 						fmt.Printf("%s,%s,%s,%s\n", distribution.Name(), archName, imgType.Name(), itConfig.Name)
 					} else {
 						var repos []rpmmd.RepoConfig
-						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot, bootcRemote, bootcInstallerRef)
+						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot, bootcRemote, bootcInstallerRef, cs)
 						jobs = append(jobs, job)
 					}
 				}
@@ -731,7 +752,7 @@ func main() {
 						}
 
 						var repos []rpmmd.RepoConfig
-						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot, bootcRemote, bootcInstallerRef)
+						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot, bootcRemote, bootcInstallerRef, cs)
 						jobs = append(jobs, job)
 					}
 				}
@@ -767,6 +788,11 @@ func main() {
 			fmt.Fprintf(os.Stderr, "\nStack trace of the first error:\n%s\n", err1.stack)
 		}
 		exit = 1
+	} else if cs != nil {
+		if err := cs.deleteStaleChecksums(); err != nil {
+			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+			exit = 1
+		}
 	}
 	fmt.Fprintf(os.Stderr, "RPM metadata cache kept in %s\n", cacheRoot)
 	os.Exit(exit)