Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 52 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ The proxy can be configured via:
-config string Path to configuration file
-listen string Address to listen on (default ":8080")
-base-url string Public URL of this proxy (default "http://localhost:8080")
-storage-url string Storage URL (file:// or s3://)
-storage-url string Storage URL (file://, s3://, gs://, azblob://)
-storage-path string Path to artifact storage directory (deprecated, use -storage-url)
-database-driver string Database driver: sqlite or postgres (default "sqlite")
-database-path string Path to SQLite database file (default "./cache/proxy.db")
Expand Down Expand Up @@ -549,6 +549,57 @@ storage:

Set credentials via standard AWS environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`).

### Google Cloud Storage

The proxy can store cached artifacts in a GCS bucket using the `gs://` URL scheme.

```yaml
storage:
url: "gs://my-bucket-name"
```

Authentication uses [Application Default Credentials](https://docs.cloud.google.com/docs/authentication/application-default-credentials), which means no credentials need to be embedded in the config or environment. Supported sources, in order:

- **GKE Workload Identity** — bind the Kubernetes service account running the proxy to a Google service account that has `roles/storage.objectAdmin` on the bucket. The proxy will use the workload's token automatically.
- **Attached service account** on GCE, Cloud Run, Cloud Functions, etc.
- **`GOOGLE_APPLICATION_CREDENTIALS`** environment variable pointing at a service account JSON key file.
- **`gcloud auth application-default login`** for local development.

#### GKE Workload Identity setup

```bash
# 1. Create a Google service account
gcloud iam service-accounts create git-pkgs-proxy \
--project=PROJECT_ID

# 2. Grant it access to the bucket
gsutil iam ch \
serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com:objectAdmin \
gs://my-bucket-name

# 3. Bind the Kubernetes service account to it
gcloud iam service-accounts add-iam-policy-binding \
git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \
--role=roles/iam.workloadIdentityUser \
--member="serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/KSA_NAME]"

# 4. Annotate the Kubernetes service account
kubectl annotate serviceaccount KSA_NAME \
--namespace=NAMESPACE \
iam.gke.io/gcp-service-account=git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com
```

#### Direct serve (signed URLs) with Workload Identity

When `direct_serve: true` is enabled, the proxy issues HTTP 302 redirects to presigned GCS URLs. Workload Identity provides no private key, so the GCS backend calls the [IAM Credentials `signBlob` API](https://docs.cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob). Grant the service account the token-creator role on itself:

```bash
gcloud iam service-accounts add-iam-policy-binding \
git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \
--role=roles/iam.serviceAccountTokenCreator \
--member="serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com"
```

## CLI Commands

### serve (default)
Expand Down
13 changes: 12 additions & 1 deletion config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,20 @@ storage:
# - file:///path/to/dir - Local filesystem (default)
# - s3://bucket-name - Amazon S3
# - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
# - gs://bucket-name - Google Cloud Storage
# - azblob://container-name - Azure Blob Storage
#
# For S3, configure credentials via environment variables:
# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
#
# For GCS, authentication uses Application Default Credentials. On GKE with
# Workload Identity, bind the Kubernetes service account to a Google service
# account that has roles/storage.objectAdmin on the bucket. No extra config
# is needed in this file. For local development, run:
# gcloud auth application-default login
# If direct_serve is enabled, the service account also needs
# roles/iam.serviceAccountTokenCreator on itself so the IAM Credentials
# signBlob API can sign URLs without a private key.
url: ""

# Local filesystem path (used when url is empty)
Expand All @@ -42,7 +53,7 @@ storage:
max_size: ""

# Redirect cached artifact downloads to presigned storage URLs (HTTP 302)
# instead of streaming through the proxy. Only effective for S3 and Azure.
# instead of streaming through the proxy. Only effective for S3, GCS, and Azure.
# Leave disabled if clients reach the proxy through an authenticating gateway,
# since presigned URLs bypass it.
direct_serve: false
Expand Down
5 changes: 2 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
module github.com/git-pkgs/proxy

go 1.26.0

toolchain go1.26.6
go 1.26.7

require (
github.com/BurntSushi/toml v1.6.0
github.com/CycloneDX/cyclonedx-go v0.11.0
github.com/git-pkgs/archives v0.5.1
github.com/git-pkgs/cooldown v0.2.0
github.com/git-pkgs/enrichment v0.7.0
github.com/git-pkgs/gcs v0.1.0
github.com/git-pkgs/integrity v0.1.1
github.com/git-pkgs/magic v0.2.0
github.com/git-pkgs/purl v0.1.19
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,8 @@ github.com/git-pkgs/cooldown v0.2.0 h1:0MWPHtkzZgvCR0wdiQeyvMea/dxgw9tParH1zzaFo
github.com/git-pkgs/cooldown v0.2.0/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E=
github.com/git-pkgs/enrichment v0.7.0 h1:LfIzlVArc2p0MONO08ybC5jiHlysfzS3YyZ5ry2d6Lw=
github.com/git-pkgs/enrichment v0.7.0/go.mod h1:ZgZJq7cz1H/nlkgHjCdIRLF/TA5VcbUmpE970VQmG14=
github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo=
github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE=
github.com/git-pkgs/integrity v0.1.1 h1:nHQ7SktOiGM1dOb5BFnkdtttG/6FCgE6r5ru6QnsGts=
github.com/git-pkgs/integrity v0.1.1/go.mod h1:hxu24lcd230377hCF28JQW7sGcCbuNLqo/0ULeb+F1Q=
github.com/git-pkgs/magic v0.2.0 h1:c7HqVxnP8c88EaVMH0/KraDFVTcmiXckRiSvNZEnvMQ=
Expand Down
16 changes: 15 additions & 1 deletion internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,22 @@
// storage:
// url: "s3://bucket?endpoint=http://localhost:9000"
//
// Google Cloud Storage:
//
// storage:
// url: "gs://bucket-name"
//
// For S3, configure credentials via AWS environment variables:
//
// AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
//
// For GCS, authentication uses Application Default Credentials. This supports
// GKE Workload Identity, attached service accounts on GCE and Cloud Run, and
// local credentials created by `gcloud auth application-default login`.
// When direct_serve is enabled without a private key, the GCS backend uses the
// IAM Credentials signBlob API. The service account must hold
// roles/iam.serviceAccountTokenCreator on itself.
//
// Database Configuration:
//
// The proxy supports two database backends:
Expand Down Expand Up @@ -181,6 +193,8 @@ type StorageConfig struct {
// - file:///path/to/dir - Local filesystem (default)
// - s3://bucket-name - Amazon S3
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
// - gs://bucket-name - Google Cloud Storage (Workload Identity supported)
// - azblob://container-name - Azure Blob Storage
// If empty, defaults to file:// with the Path value.
URL string `json:"url" yaml:"url"`

Expand All @@ -197,7 +211,7 @@ type StorageConfig struct {

// DirectServe enables redirecting cached artifact downloads to presigned
// storage URLs (HTTP 302) instead of streaming bytes through the proxy.
// Only effective for backends that support URL signing (S3, Azure).
// Only effective for backends that support URL signing (S3, GCS, Azure).
DirectServe bool `json:"direct_serve" yaml:"direct_serve"`

// DirectServeTTL is how long presigned URLs remain valid.
Expand Down
7 changes: 6 additions & 1 deletion internal/server/eviction_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,17 @@ func setupEvictionTest(t *testing.T) (*database.DB, *storage.Blob) {
_ = db.Close()
t.Fatalf("failed to create storage: %v", err)
}
blob, ok := store.(*storage.Blob)
if !ok {
_ = db.Close()
t.Fatalf("OpenBucket returned %T, want *storage.Blob", store)
}

t.Cleanup(func() {
_ = db.Close()
})

return db, store
return db, blob
}

func seedArtifact(t *testing.T, ctx context.Context, db *database.DB, store storage.Storage, name string, dataSize int, accessedAt time.Time) {
Expand Down
11 changes: 10 additions & 1 deletion internal/storage/blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,18 @@ type Blob struct {
// - file:///path/to/dir - Local filesystem storage
// - s3://bucket-name - Amazon S3 (uses AWS_* environment variables)
// - s3://bucket-name?region=us-east-1&endpoint=http://localhost:9000 - S3-compatible (MinIO, etc.)
// - gs://bucket-name - Google Cloud Storage (uses Application Default Credentials;
// supports Workload Identity on GKE/GCE without any extra configuration)
// - azblob://container-name - Azure Blob Storage
//
// For local filesystem, the directory is created if it doesn't exist.
func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) {
//
//nolint:ireturn // The URL scheme selects the storage implementation.
func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
if strings.HasPrefix(urlStr, "gs://") {
return OpenGCS(ctx, urlStr)
}

// Handle file:// URLs specially to create the directory
if strings.HasPrefix(urlStr, "file://") {
path := strings.TrimPrefix(urlStr, "file://")
Expand Down
6 changes: 5 additions & 1 deletion internal/storage/blob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,11 @@ func createTestBlob(t *testing.T) *Blob {
t.Fatalf("OpenBucket failed: %v", err)
}
t.Cleanup(func() { _ = b.Close() })
return b
blob, ok := b.(*Blob)
if !ok {
t.Fatalf("OpenBucket returned %T, want *Blob", b)
}
return blob
}

func fileURLFromPath(path string) string {
Expand Down
97 changes: 97 additions & 0 deletions internal/storage/gcs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package storage

import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"time"

gcstorage "github.com/git-pkgs/gcs"
)

// GCS adapts a Google Cloud Storage bucket to Storage.
type GCS struct {
bucket *gcstorage.Bucket
url string
}

// OpenGCS opens a Google Cloud Storage bucket from a gs:// URL.
func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) {
bucket, err := gcstorage.OpenBucket(ctx, urlStr)
if err != nil {
return nil, err
}
return &GCS{bucket: bucket, url: urlStr}, nil
}

func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
h := sha256.New()
size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h))
if err != nil {
return 0, "", err
}
return size, hex.EncodeToString(h.Sum(nil)), nil
}

func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) {
r, err := g.bucket.Open(ctx, path)
if errors.Is(err, gcstorage.ErrNotFound) {
return nil, ErrNotFound
}
return r, err
}

func (g *GCS) Exists(ctx context.Context, path string) (bool, error) {
return g.bucket.Exists(ctx, path)
}

func (g *GCS) Delete(ctx context.Context, path string) error {
return g.bucket.Delete(ctx, path)
}

func (g *GCS) Size(ctx context.Context, path string) (int64, error) {
size, err := g.bucket.Size(ctx, path)
if errors.Is(err, gcstorage.ErrNotFound) {
return 0, ErrNotFound
}
return size, err
}

func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
u, err := g.bucket.SignedURL(ctx, path, expiry)
if errors.Is(err, gcstorage.ErrSignedURLUnsupported) {
return "", ErrSignedURLUnsupported
}
return u, err
}

func (g *GCS) UsedSpace(ctx context.Context) (int64, error) {
return g.bucket.UsedSpace(ctx)
}

func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
objects, err := g.bucket.ListPrefix(ctx, prefix)
if err != nil {
return nil, err
}

result := make([]ObjectInfo, 0, len(objects))
for _, object := range objects {
result = append(result, ObjectInfo{
Path: object.Name,
Size: object.Size,
ModTime: object.ModTime,
})
}
return result, nil
}

func (g *GCS) Close() error {
return nil
}

func (g *GCS) URL() string {
return g.url
}
Loading