diff --git a/README.md b/README.md index 320a737..163f3d6 100644 --- a/README.md +++ b/README.md @@ -449,7 +449,7 @@ The proxy can be configured via: -config string Path to configuration file -listen string Address to listen on (default ":8080") -base-url string Public URL of this proxy (default "http://localhost:8080") --storage-url string Storage URL (file:// or s3://) +-storage-url string Storage URL (file://, s3://, gs://, azblob://) -storage-path string Path to artifact storage directory (deprecated, use -storage-url) -database-driver string Database driver: sqlite or postgres (default "sqlite") -database-path string Path to SQLite database file (default "./cache/proxy.db") @@ -549,6 +549,57 @@ storage: Set credentials via standard AWS environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`). +### Google Cloud Storage + +The proxy can store cached artifacts in a GCS bucket using the `gs://` URL scheme. + +```yaml +storage: + url: "gs://my-bucket-name" +``` + +Authentication uses [Application Default Credentials](https://docs.cloud.google.com/docs/authentication/application-default-credentials), which means no credentials need to be embedded in the config or environment. Supported sources, in order: + +- **GKE Workload Identity** — bind the Kubernetes service account running the proxy to a Google service account that has `roles/storage.objectAdmin` on the bucket. The proxy will use the workload's token automatically. +- **Attached service account** on GCE, Cloud Run, Cloud Functions, etc. +- **`GOOGLE_APPLICATION_CREDENTIALS`** environment variable pointing at a service account JSON key file. +- **`gcloud auth application-default login`** for local development. + +#### GKE Workload Identity setup + +```bash +# 1. Create a Google service account +gcloud iam service-accounts create git-pkgs-proxy \ + --project=PROJECT_ID + +# 2. Grant it access to the bucket +gsutil iam ch \ + serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com:objectAdmin \ + gs://my-bucket-name + +# 3. Bind the Kubernetes service account to it +gcloud iam service-accounts add-iam-policy-binding \ + git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/iam.workloadIdentityUser \ + --member="serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/KSA_NAME]" + +# 4. Annotate the Kubernetes service account +kubectl annotate serviceaccount KSA_NAME \ + --namespace=NAMESPACE \ + iam.gke.io/gcp-service-account=git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com +``` + +#### Direct serve (signed URLs) with Workload Identity + +When `direct_serve: true` is enabled, the proxy issues HTTP 302 redirects to presigned GCS URLs. Workload Identity provides no private key, so the GCS backend calls the [IAM Credentials `signBlob` API](https://docs.cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob). Grant the service account the token-creator role on itself: + +```bash +gcloud iam service-accounts add-iam-policy-binding \ + git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/iam.serviceAccountTokenCreator \ + --member="serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com" +``` + ## CLI Commands ### serve (default) diff --git a/config.example.yaml b/config.example.yaml index 1df95b3..b4470a8 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -27,9 +27,20 @@ storage: # - file:///path/to/dir - Local filesystem (default) # - s3://bucket-name - Amazon S3 # - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) + # - gs://bucket-name - Google Cloud Storage + # - azblob://container-name - Azure Blob Storage # # For S3, configure credentials via environment variables: # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION + # + # For GCS, authentication uses Application Default Credentials. On GKE with + # Workload Identity, bind the Kubernetes service account to a Google service + # account that has roles/storage.objectAdmin on the bucket. No extra config + # is needed in this file. For local development, run: + # gcloud auth application-default login + # If direct_serve is enabled, the service account also needs + # roles/iam.serviceAccountTokenCreator on itself so the IAM Credentials + # signBlob API can sign URLs without a private key. url: "" # Local filesystem path (used when url is empty) @@ -42,7 +53,7 @@ storage: max_size: "" # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) - # instead of streaming through the proxy. Only effective for S3 and Azure. + # instead of streaming through the proxy. Only effective for S3, GCS, and Azure. # Leave disabled if clients reach the proxy through an authenticating gateway, # since presigned URLs bypass it. direct_serve: false diff --git a/go.mod b/go.mod index c4c740b..0203ed2 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/git-pkgs/proxy -go 1.26.0 - -toolchain go1.26.6 +go 1.26.7 require ( github.com/BurntSushi/toml v1.6.0 @@ -10,6 +8,7 @@ require ( github.com/git-pkgs/archives v0.5.1 github.com/git-pkgs/cooldown v0.2.0 github.com/git-pkgs/enrichment v0.7.0 + github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.1.1 github.com/git-pkgs/magic v0.2.0 github.com/git-pkgs/purl v0.1.19 diff --git a/go.sum b/go.sum index 2cdf33b..5e9e2cc 100644 --- a/go.sum +++ b/go.sum @@ -263,6 +263,8 @@ github.com/git-pkgs/cooldown v0.2.0 h1:0MWPHtkzZgvCR0wdiQeyvMea/dxgw9tParH1zzaFo github.com/git-pkgs/cooldown v0.2.0/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= github.com/git-pkgs/enrichment v0.7.0 h1:LfIzlVArc2p0MONO08ybC5jiHlysfzS3YyZ5ry2d6Lw= github.com/git-pkgs/enrichment v0.7.0/go.mod h1:ZgZJq7cz1H/nlkgHjCdIRLF/TA5VcbUmpE970VQmG14= +github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= +github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.1.1 h1:nHQ7SktOiGM1dOb5BFnkdtttG/6FCgE6r5ru6QnsGts= github.com/git-pkgs/integrity v0.1.1/go.mod h1:hxu24lcd230377hCF28JQW7sGcCbuNLqo/0ULeb+F1Q= github.com/git-pkgs/magic v0.2.0 h1:c7HqVxnP8c88EaVMH0/KraDFVTcmiXckRiSvNZEnvMQ= diff --git a/internal/config/config.go b/internal/config/config.go index a3dfbc6..bedac93 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,10 +24,22 @@ // storage: // url: "s3://bucket?endpoint=http://localhost:9000" // +// Google Cloud Storage: +// +// storage: +// url: "gs://bucket-name" +// // For S3, configure credentials via AWS environment variables: // // AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION // +// For GCS, authentication uses Application Default Credentials. This supports +// GKE Workload Identity, attached service accounts on GCE and Cloud Run, and +// local credentials created by `gcloud auth application-default login`. +// When direct_serve is enabled without a private key, the GCS backend uses the +// IAM Credentials signBlob API. The service account must hold +// roles/iam.serviceAccountTokenCreator on itself. +// // Database Configuration: // // The proxy supports two database backends: @@ -181,6 +193,8 @@ type StorageConfig struct { // - file:///path/to/dir - Local filesystem (default) // - s3://bucket-name - Amazon S3 // - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) + // - gs://bucket-name - Google Cloud Storage (Workload Identity supported) + // - azblob://container-name - Azure Blob Storage // If empty, defaults to file:// with the Path value. URL string `json:"url" yaml:"url"` @@ -197,7 +211,7 @@ type StorageConfig struct { // DirectServe enables redirecting cached artifact downloads to presigned // storage URLs (HTTP 302) instead of streaming bytes through the proxy. - // Only effective for backends that support URL signing (S3, Azure). + // Only effective for backends that support URL signing (S3, GCS, Azure). DirectServe bool `json:"direct_serve" yaml:"direct_serve"` // DirectServeTTL is how long presigned URLs remain valid. diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go index 80badbe..bac3325 100644 --- a/internal/server/eviction_test.go +++ b/internal/server/eviction_test.go @@ -32,12 +32,17 @@ func setupEvictionTest(t *testing.T) (*database.DB, *storage.Blob) { _ = db.Close() t.Fatalf("failed to create storage: %v", err) } + blob, ok := store.(*storage.Blob) + if !ok { + _ = db.Close() + t.Fatalf("OpenBucket returned %T, want *storage.Blob", store) + } t.Cleanup(func() { _ = db.Close() }) - return db, store + return db, blob } func seedArtifact(t *testing.T, ctx context.Context, db *database.DB, store storage.Storage, name string, dataSize int, accessedAt time.Time) { diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 67e91d0..97e50f3 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -35,9 +35,18 @@ type Blob struct { // - file:///path/to/dir - Local filesystem storage // - s3://bucket-name - Amazon S3 (uses AWS_* environment variables) // - s3://bucket-name?region=us-east-1&endpoint=http://localhost:9000 - S3-compatible (MinIO, etc.) +// - gs://bucket-name - Google Cloud Storage (uses Application Default Credentials; +// supports Workload Identity on GKE/GCE without any extra configuration) +// - azblob://container-name - Azure Blob Storage // // For local filesystem, the directory is created if it doesn't exist. -func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) { +// +//nolint:ireturn // The URL scheme selects the storage implementation. +func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { + if strings.HasPrefix(urlStr, "gs://") { + return OpenGCS(ctx, urlStr) + } + // Handle file:// URLs specially to create the directory if strings.HasPrefix(urlStr, "file://") { path := strings.TrimPrefix(urlStr, "file://") diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index d80290b..3e5bf65 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -278,7 +278,11 @@ func createTestBlob(t *testing.T) *Blob { t.Fatalf("OpenBucket failed: %v", err) } t.Cleanup(func() { _ = b.Close() }) - return b + blob, ok := b.(*Blob) + if !ok { + t.Fatalf("OpenBucket returned %T, want *Blob", b) + } + return blob } func fileURLFromPath(path string) string { diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go new file mode 100644 index 0000000..a096c18 --- /dev/null +++ b/internal/storage/gcs.go @@ -0,0 +1,97 @@ +package storage + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "time" + + gcstorage "github.com/git-pkgs/gcs" +) + +// GCS adapts a Google Cloud Storage bucket to Storage. +type GCS struct { + bucket *gcstorage.Bucket + url string +} + +// OpenGCS opens a Google Cloud Storage bucket from a gs:// URL. +func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) { + bucket, err := gcstorage.OpenBucket(ctx, urlStr) + if err != nil { + return nil, err + } + return &GCS{bucket: bucket, url: urlStr}, nil +} + +func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + h := sha256.New() + size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h)) + if err != nil { + return 0, "", err + } + return size, hex.EncodeToString(h.Sum(nil)), nil +} + +func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { + r, err := g.bucket.Open(ctx, path) + if errors.Is(err, gcstorage.ErrNotFound) { + return nil, ErrNotFound + } + return r, err +} + +func (g *GCS) Exists(ctx context.Context, path string) (bool, error) { + return g.bucket.Exists(ctx, path) +} + +func (g *GCS) Delete(ctx context.Context, path string) error { + return g.bucket.Delete(ctx, path) +} + +func (g *GCS) Size(ctx context.Context, path string) (int64, error) { + size, err := g.bucket.Size(ctx, path) + if errors.Is(err, gcstorage.ErrNotFound) { + return 0, ErrNotFound + } + return size, err +} + +func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { + u, err := g.bucket.SignedURL(ctx, path, expiry) + if errors.Is(err, gcstorage.ErrSignedURLUnsupported) { + return "", ErrSignedURLUnsupported + } + return u, err +} + +func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { + return g.bucket.UsedSpace(ctx) +} + +func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { + objects, err := g.bucket.ListPrefix(ctx, prefix) + if err != nil { + return nil, err + } + + result := make([]ObjectInfo, 0, len(objects)) + for _, object := range objects { + result = append(result, ObjectInfo{ + Path: object.Name, + Size: object.Size, + ModTime: object.ModTime, + }) + } + return result, nil +} + +func (g *GCS) Close() error { + return nil +} + +func (g *GCS) URL() string { + return g.url +} diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go new file mode 100644 index 0000000..1dfd0a2 --- /dev/null +++ b/internal/storage/gcs_test.go @@ -0,0 +1,152 @@ +package storage + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "strconv" + "strings" + "testing" + "time" +) + +func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) { + server := httptest.NewServer(&fakeGCSServer{t: t, objects: map[string]string{}}) + defer server.Close() + t.Setenv("STORAGE_EMULATOR_HOST", server.URL) + + ctx := context.Background() + store, err := OpenBucket(ctx, "gs://test-bucket") + if err != nil { + t.Fatalf("OpenBucket failed: %v", err) + } + + size, hash, err := store.Store(ctx, "npm/pkg/file.tgz", strings.NewReader("content")) + if err != nil { + t.Fatalf("Store failed: %v", err) + } + wantHash := sha256.Sum256([]byte("content")) + if size != int64(len("content")) || hash != hex.EncodeToString(wantHash[:]) { + t.Fatalf("Store returned size=%d hash=%q", size, hash) + } + + exists, err := store.Exists(ctx, "npm/pkg/file.tgz") + if err != nil || !exists { + t.Fatalf("Exists = %v, %v; want true, nil", exists, err) + } + + r, err := store.Open(ctx, "npm/pkg/file.tgz") + if err != nil { + t.Fatalf("Open failed: %v", err) + } + data, _ := io.ReadAll(r) + _ = r.Close() + if string(data) != "content" { + t.Fatalf("Open content = %q, want content", data) + } + + lister, ok := store.(interface { + ListPrefix(context.Context, string) ([]ObjectInfo, error) + }) + if !ok { + t.Fatal("GCS storage does not support prefix listing") + } + list, err := lister.ListPrefix(ctx, "npm/") + if err != nil { + t.Fatalf("ListPrefix failed: %v", err) + } + if len(list) != 1 || list[0].Path != "npm/pkg/file.tgz" { + t.Fatalf("ListPrefix = %#v", list) + } + + if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil { + t.Fatalf("Delete failed: %v", err) + } + exists, err = store.Exists(ctx, "npm/pkg/file.tgz") + if err != nil || exists { + t.Fatalf("Exists after delete = %v, %v; want false, nil", exists, err) + } + + reader, err := store.Open(ctx, "npm/pkg/file.tgz") + if reader != nil || !errors.Is(err, ErrNotFound) { + t.Fatalf("Open missing object = %v, %v; want nil, ErrNotFound", reader, err) + } + if _, err := store.Size(ctx, "npm/pkg/file.tgz"); !errors.Is(err, ErrNotFound) { + t.Fatalf("Size missing object = %v, want ErrNotFound", err) + } + if _, err := store.SignedURL(ctx, "npm/pkg/file.tgz", time.Minute); !errors.Is(err, ErrSignedURLUnsupported) { + t.Fatalf("SignedURL with emulator = %v, want ErrSignedURLUnsupported", err) + } +} + +type fakeGCSServer struct { + t *testing.T + objects map[string]string +} + +func (f *fakeGCSServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/upload/storage/v1/b/test-bucket/o": + name := r.URL.Query().Get("name") + data, _ := io.ReadAll(r.Body) + f.objects[name] = string(data) + writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) + case r.Method == http.MethodGet && r.URL.Path == "/storage/v1/b/test-bucket/o": + prefix := r.URL.Query().Get("prefix") + page := fakeGCSListResponse{} + for name, data := range f.objects { + if strings.HasPrefix(name, prefix) { + page.Items = append(page.Items, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) + } + } + sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].Name < page.Items[j].Name }) + writeJSON(w, page) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"): + name := objectNameFromPath(r.URL.Path) + data, ok := f.objects[name] + if !ok { + http.NotFound(w, r) + return + } + if r.URL.Query().Get("alt") == "media" { + _, _ = io.WriteString(w, data) + return + } + writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"): + delete(f.objects, objectNameFromPath(r.URL.Path)) + w.WriteHeader(http.StatusNoContent) + default: + f.t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } +} + +type fakeGCSObject struct { + Name string `json:"name"` + Size string `json:"size"` + Updated string `json:"updated"` +} + +type fakeGCSListResponse struct { + NextPageToken string `json:"nextPageToken"` + Items []fakeGCSObject `json:"items"` +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) +} + +func objectNameFromPath(p string) string { + escaped := strings.TrimPrefix(p, "/storage/v1/b/test-bucket/o/") + name, _ := url.PathUnescape(escaped) + return name +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 3d0be1c..5ff86f2 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -5,6 +5,9 @@ // - file:///path/to/dir - Local filesystem storage // - s3://bucket-name - Amazon S3 // - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) +// - gs://bucket-name - Google Cloud Storage (supports GKE Workload Identity +// via Application Default Credentials) +// - azblob://container-name - Azure Blob Storage // // Use OpenBucket to create a storage backend from a URL. package storage