diff --git a/controllers/dependencyfirewall/cache_stats.go b/controllers/dependencyfirewall/cache_stats.go new file mode 100644 index 000000000..c8775ed5b --- /dev/null +++ b/controllers/dependencyfirewall/cache_stats.go @@ -0,0 +1,189 @@ +// Copyright (C) 2026 l3montree GmbH +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dependencyfirewall + +import ( + "errors" + "fmt" + "io/fs" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/l3montree-dev/devguard/shared" + "github.com/labstack/echo/v4" +) + +// CacheStats describes aggregate storage usage for a slice of the dependency +// proxy on-disk cache (one ecosystem, the OCI manifest/blob split, or the +// overall total). +type CacheStats struct { + SizeBytes int64 `json:"sizeBytes"` + Entries int `json:"entries"` + OldestEntry *time.Time `json:"oldestEntry,omitempty"` + NewestEntry *time.Time `json:"newestEntry,omitempty"` +} + +// OCISubStats further splits the OCI cache into manifests and blobs. +type OCISubStats struct { + Manifests CacheStats `json:"manifests"` + Blobs CacheStats `json:"blobs"` +} + +// CacheStatsResponse is the JSON returned by GetCacheStats. +type CacheStatsResponse struct { + CacheDir string `json:"cacheDir"` + TotalSize int64 `json:"totalSizeBytes"` + TotalEntries int `json:"totalEntries"` + OldestEntry *time.Time `json:"oldestEntry,omitempty"` + NewestEntry *time.Time `json:"newestEntry,omitempty"` + ByEcosystem map[string]CacheStats `json:"byEcosystem"` + OCIBreakdown OCISubStats `json:"ociBreakdown"` +} + +// companionSuffixes are sidecar files written next to a cached payload. They +// count toward disk usage but not toward entry counts. +var companionSuffixes = []string{".sha256", ".releasetime", ".contenttype", ".digest"} + +func isCompanionFile(name string) bool { + for _, s := range companionSuffixes { + if strings.HasSuffix(name, s) { + return true + } + } + return false +} + +// GetCacheStats walks the on-disk dependency proxy cache and returns aggregate +// storage statistics, both overall and broken down per ecosystem. The OCI +// subtree is split further into manifests vs blobs. +func (d *DependencyProxyController) GetCacheStats(ctx shared.Context) error { + resp, err := d.CollectCacheStats() + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to read cache stats: %v", err)) + } + return ctx.JSON(http.StatusOK, resp) +} + +func (d *DependencyProxyController) CollectCacheStats() (CacheStatsResponse, error) { + resp := CacheStatsResponse{ + CacheDir: d.cacheDir, + ByEcosystem: map[string]CacheStats{}, + } + + if _, err := os.Stat(d.cacheDir); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return resp, nil + } + return resp, err + } + + ecoStats := map[string]*CacheStats{} + total := &CacheStats{} + ociMan := &CacheStats{} + ociBlob := &CacheStats{} + + walkErr := filepath.Walk(d.cacheDir, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + rel, relErr := filepath.Rel(d.cacheDir, path) + if relErr != nil { + return nil + } + relSlash := filepath.ToSlash(rel) + parts := strings.SplitN(relSlash, "/", 2) + if len(parts) < 2 { + // File directly under cache root — not produced by the proxy code paths, + // but keep accounting honest by including its size in the total. + total.SizeBytes += info.Size() + updateTimes(total, info.ModTime()) + return nil + } + eco := parts[0] + stats, ok := ecoStats[eco] + if !ok { + stats = &CacheStats{} + ecoStats[eco] = stats + } + + size := info.Size() + mod := info.ModTime() + companion := isCompanionFile(info.Name()) + + stats.SizeBytes += size + updateTimes(stats, mod) + if !companion { + stats.Entries++ + } + + total.SizeBytes += size + updateTimes(total, mod) + if !companion { + total.Entries++ + } + + if eco == "oci" { + switch { + case strings.Contains(relSlash, "/manifests/"): + ociMan.SizeBytes += size + updateTimes(ociMan, mod) + if !companion { + ociMan.Entries++ + } + case strings.Contains(relSlash, "/blobs/"): + ociBlob.SizeBytes += size + updateTimes(ociBlob, mod) + if !companion { + ociBlob.Entries++ + } + } + } + + return nil + }) + if walkErr != nil { + return resp, walkErr + } + + for eco, s := range ecoStats { + resp.ByEcosystem[eco] = *s + } + resp.TotalSize = total.SizeBytes + resp.TotalEntries = total.Entries + resp.OldestEntry = total.OldestEntry + resp.NewestEntry = total.NewestEntry + resp.OCIBreakdown = OCISubStats{Manifests: *ociMan, Blobs: *ociBlob} + + return resp, nil +} + +func updateTimes(s *CacheStats, mod time.Time) { + if s.OldestEntry == nil || mod.Before(*s.OldestEntry) { + t := mod + s.OldestEntry = &t + } + if s.NewestEntry == nil || mod.After(*s.NewestEntry) { + t := mod + s.NewestEntry = &t + } +} diff --git a/router/org_router.go b/router/org_router.go index 5a1f9eb1c..5419c6745 100644 --- a/router/org_router.go +++ b/router/org_router.go @@ -66,6 +66,7 @@ func NewOrgRouter( organizationRouter.GET("/config-files/:config-file/", orgController.GetConfigFile) organizationRouter.GET("/dependency-proxy-urls/", dependencyProxyController.GetDependencyProxyURLs) + organizationRouter.GET("/dependency-proxy/cache-stats/", dependencyProxyController.GetCacheStats, middlewares.NeededScope([]string{"manage"}), middlewares.OrganizationAccessControlMiddleware(shared.ObjectOrganization, shared.ActionUpdate)) // admin-only: storage consumption + per-ecosystem breakdown of the shared dep/oci proxy cache organizationRouter.PUT("/config-files/:config-file/", orgController.UpdateConfigFile, middlewares.NeededScope([]string{"manage"}), middlewares.OrganizationAccessControlMiddleware(shared.ObjectOrganization, shared.ActionUpdate)) organizationRouter.GET("/trigger-sync/", externalEntityProviderService.TriggerSync) organizationRouter.GET("/settings/", orgController.AdminSettings, middlewares.NeededScope([]string{"manage"}), middlewares.OrganizationAccessControlMiddleware(shared.ObjectOrganization, shared.ActionUpdate)) diff --git a/tests/dependency_proxy_controller_test.go b/tests/dependency_proxy_controller_test.go index db06a3fab..34153fb90 100644 --- a/tests/dependency_proxy_controller_test.go +++ b/tests/dependency_proxy_controller_test.go @@ -155,6 +155,102 @@ func TestDependencyProxyControllerMaliciousPackageRemoval(t *testing.T) { }) } +func TestDependencyProxyControllerCacheStats(t *testing.T) { + tempDir := t.TempDir() + + config := dependencyfirewall.DependencyProxyCache{ + CacheDir: tempDir, + } + + checker, err := vulndb.NewMaliciousPackageChecker(nil) + require.NoError(t, err) + + controller := dependencyfirewall.NewDependencyProxyController(nil, config, checker, nil, nil, nil) + + writeFile := func(rel string, size int) { + path := filepath.Join(tempDir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + buf := make([]byte, size) + require.NoError(t, os.WriteFile(path, buf, 0644)) + } + + // Layout mirrors what the proxy actually writes: + // //[.companion] + writeFile("npm/lodash/-/lodash-4.17.21.tgz", 1000) + writeFile("npm/lodash/-/lodash-4.17.21.tgz.sha256", 64) + writeFile("npm/lodash/-/lodash-4.17.21.tgz.releasetime", 30) + writeFile("go/cloud.google.com/go/@v/v0.110.0.zip", 2000) + writeFile("pypi/packages/abc/requests-2.31.0.tar.gz", 500) + writeFile("oci/docker.io/library/nginx/manifests/latest", 800) + writeFile("oci/docker.io/library/nginx/manifests/latest.contenttype", 40) + writeFile("oci/docker.io/library/nginx/manifests/latest.digest", 71) + writeFile("oci/docker.io/library/nginx/blobs/sha256_aaa", 4000) + writeFile("oci/docker.io/library/nginx/blobs/sha256_aaa.sha256", 64) + + stats, err := controller.CollectCacheStats() + require.NoError(t, err) + + t.Run("cache dir is reported", func(t *testing.T) { + assert.Equal(t, tempDir, stats.CacheDir) + }) + + t.Run("totals sum every file but count only payloads", func(t *testing.T) { + // 1000+64+30 + 2000 + 500 + 800+40+71 + 4000+64 = 8569 + assert.Equal(t, int64(8569), stats.TotalSize) + // payloads: lodash tgz, go zip, requests tar, oci manifest, oci blob = 5 + assert.Equal(t, 5, stats.TotalEntries) + }) + + t.Run("per-ecosystem breakdown", func(t *testing.T) { + require.Contains(t, stats.ByEcosystem, "npm") + require.Contains(t, stats.ByEcosystem, "go") + require.Contains(t, stats.ByEcosystem, "pypi") + require.Contains(t, stats.ByEcosystem, "oci") + + assert.Equal(t, int64(1094), stats.ByEcosystem["npm"].SizeBytes) + assert.Equal(t, 1, stats.ByEcosystem["npm"].Entries) + + assert.Equal(t, int64(2000), stats.ByEcosystem["go"].SizeBytes) + assert.Equal(t, 1, stats.ByEcosystem["go"].Entries) + + assert.Equal(t, int64(500), stats.ByEcosystem["pypi"].SizeBytes) + assert.Equal(t, 1, stats.ByEcosystem["pypi"].Entries) + + assert.Equal(t, int64(4975), stats.ByEcosystem["oci"].SizeBytes) + assert.Equal(t, 2, stats.ByEcosystem["oci"].Entries) + }) + + t.Run("oci breakdown splits manifests vs blobs", func(t *testing.T) { + assert.Equal(t, int64(911), stats.OCIBreakdown.Manifests.SizeBytes) + assert.Equal(t, 1, stats.OCIBreakdown.Manifests.Entries) + assert.Equal(t, int64(4064), stats.OCIBreakdown.Blobs.SizeBytes) + assert.Equal(t, 1, stats.OCIBreakdown.Blobs.Entries) + }) + + t.Run("oldest and newest entry timestamps are set", func(t *testing.T) { + require.NotNil(t, stats.OldestEntry) + require.NotNil(t, stats.NewestEntry) + assert.False(t, stats.OldestEntry.After(*stats.NewestEntry)) + }) +} + +func TestDependencyProxyControllerCacheStatsMissingDir(t *testing.T) { + missing := filepath.Join(t.TempDir(), "does-not-exist") + config := dependencyfirewall.DependencyProxyCache{CacheDir: missing} + + checker, err := vulndb.NewMaliciousPackageChecker(nil) + require.NoError(t, err) + + controller := dependencyfirewall.NewDependencyProxyController(nil, config, checker, nil, nil, nil) + + stats, err := controller.CollectCacheStats() + require.NoError(t, err) + assert.Equal(t, missing, stats.CacheDir) + assert.Zero(t, stats.TotalSize) + assert.Zero(t, stats.TotalEntries) + assert.Empty(t, stats.ByEcosystem) +} + func TestDependencyProxyControllerExtractNPMVersion(t *testing.T) { tempDir := t.TempDir()