From 66db7a06ab316ac12432a44db4ed551460fc2778 Mon Sep 17 00:00:00 2001 From: Anchit-04 Date: Wed, 12 Aug 2026 13:15:26 +0000 Subject: [PATCH] Support air-gapped catalogs via a local ConfigMap-mounted index Right now plugin-hub always fetches index.json from GitHub, so a cluster with no outbound internet access can never populate its catalog. Add an alternate local-file source, selected by a new HUB_INDEX_PATH env var (or the chart's localIndex.configMapName value), that takes precedence over the existing hubIndexURL fetch. - catalogCache now wraps a source-agnostic fetch func instead of owning an *http.Client directly, so both the HTTP path (fetchRemoteIndex) and the new local-file path (fetchLocalIndex) share the same TTL caching and stale-on-error fallback. This also makes the cache unit-testable without a live server. - The chart mounts the ConfigMap as a directory rather than a subPath file, so updates propagate on the existing cache TTL without a pod restart. - Defaults are unchanged: with localIndex.configMapName empty, no new env var, volume, or mount is rendered. Fixes #12 Signed-off-by: Anchit-04 --- README.md | 25 ++- backend/catalog_test.go | 188 ++++++++++++++++++++ backend/main.go | 94 +++++++--- charts/plugin-hub/templates/deployment.yaml | 16 ++ charts/plugin-hub/values.yaml | 9 + 5 files changed, 308 insertions(+), 24 deletions(-) create mode 100644 backend/catalog_test.go diff --git a/README.md b/README.md index c535e37..16ccc0e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ The Hub plugin for [OpenEverest](https://github.com/openeverest/openeverest). It - **Sidebar entry + dedicated route** registered as an OpenEverest generic plugin. - **Searchable, filterable table** of all catalog entries (plugins and providers) with install status, default-channel version, categories, and a per-entry detail drawer. - **Backend-side join**: a single `/api/summary` call merges the hub catalog with the live `InstalledExtension` list so the UI does one round-trip. -- **Stale-tolerant**: the upstream catalog is cached in-memory; if GitHub becomes unreachable the last successful response is served with an `X-Hub-Stale: true` header. +- **Stale-tolerant**: the catalog is cached in-memory; if the source becomes unreachable the last successful response is served with an `X-Hub-Stale: true` header. +- **Air-gapped friendly**: point `localIndex.configMapName` (chart) or `HUB_INDEX_PATH` (backend) at a locally-provided `index.json` and the backend never has to reach GitHub for the catalog. - **Browse-only in v1.** Each catalog entry shows a copy-pasteable `helm install …` command. Install/uninstall actions from the UI will land in a later release. Built from the [`openeverest/generic-plugin-template`](https://github.com/openeverest/generic-plugin-template) and following the [Generic Plugins Architecture Design](https://github.com/openeverest/openeverest/blob/main/docs/process/generic-plugins-design.md). @@ -47,8 +48,27 @@ Edit `charts/plugin-hub/values.yaml` (or pass `--set` flags): | `plugin.enabled` | Enable/disable the plugin | `true` | | `everestAPIURL` | OpenEverest API server URL (defaults to in-cluster DNS `everest.everest-system.svc.cluster.local:8080` when empty) | `""` | | `hubIndexURL` | Override the catalog index URL | official hub when empty | +| `localIndex.configMapName` | Existing ConfigMap holding a local catalog index, for air-gapped clusters. Mounted at `/etc/plugin-hub-index` and takes precedence over `hubIndexURL` when set. | `""` | +| `localIndex.key` | Key inside that ConfigMap holding the index JSON | `index.json` | | `cacheTTLSeconds` | Catalog cache TTL (seconds) | `300` when empty | +## Air-gapped / offline catalog + +For clusters without outbound access to GitHub, provide the catalog index from a ConfigMap instead: + +```bash +kubectl create configmap plugin-hub-index \ + --from-file=index.json=./index.json \ + -n everest-system + +helm upgrade --install plugin-hub oci://ghcr.io/openeverest/charts/plugin-hub \ + --version 0.1.14 \ + -n everest-system \ + --set localIndex.configMapName=plugin-hub-index +``` + +The chart mounts the ConfigMap as a directory (not a `subPath` file), so the backend picks up updates on its normal cache TTL after `kubectl apply`-ing a new ConfigMap revision — no pod restart needed, the same way the GitHub-backed catalog already refreshes on TTL expiry. + ## Local development ```bash @@ -105,6 +125,9 @@ docker build -t plugin-hub:dev . │ │ ▼ │ │ │ GitHub raw │ │ │ hub/index.json │ + │ │ — or a mounted │ + │ │ ConfigMap file │ + │ │ (air-gapped) │ │ │ │ │ └─► OpenEverest API │ │ /v1/installed- │ diff --git a/backend/catalog_test.go b/backend/catalog_test.go new file mode 100644 index 0000000..09d105d --- /dev/null +++ b/backend/catalog_test.go @@ -0,0 +1,188 @@ +package main + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +// --------------------------------------------------------------------------- +// catalogCache +// --------------------------------------------------------------------------- + +func TestCatalogCache_ServesFreshWithinTTL(t *testing.T) { + calls := 0 + cache := newCatalogCache(time.Hour, func() ([]byte, error) { + calls++ + return []byte("body"), nil + }) + + for i := 0; i < 3; i++ { + body, stale, err := cache.get() + if err != nil { + t.Fatalf("get() #%d: unexpected error: %v", i, err) + } + if stale { + t.Fatalf("get() #%d: expected fresh, got stale", i) + } + if string(body) != "body" { + t.Fatalf("get() #%d: body = %q, want %q", i, body, "body") + } + } + if calls != 1 { + t.Fatalf("fetch called %d times within TTL, want 1", calls) + } +} + +func TestCatalogCache_RefetchesAfterTTL(t *testing.T) { + calls := 0 + cache := newCatalogCache(5*time.Millisecond, func() ([]byte, error) { + calls++ + return []byte("body"), nil + }) + + if _, _, err := cache.get(); err != nil { + t.Fatalf("first get(): %v", err) + } + time.Sleep(15 * time.Millisecond) + if _, _, err := cache.get(); err != nil { + t.Fatalf("second get(): %v", err) + } + if calls != 2 { + t.Fatalf("fetch called %d times across TTL expiry, want 2", calls) + } +} + +func TestCatalogCache_ServesStaleOnFetchError(t *testing.T) { + calls := 0 + cache := newCatalogCache(5*time.Millisecond, func() ([]byte, error) { + calls++ + if calls == 1 { + return []byte("good"), nil + } + return nil, errors.New("upstream down") + }) + + if _, _, err := cache.get(); err != nil { + t.Fatalf("first get(): %v", err) + } + time.Sleep(15 * time.Millisecond) + + body, stale, err := cache.get() + if err != nil { + t.Fatalf("get() after fetch failure: unexpected error: %v", err) + } + if !stale { + t.Fatal("get() after fetch failure: expected stale=true") + } + if string(body) != "good" { + t.Fatalf("get() after fetch failure: body = %q, want last-good %q", body, "good") + } +} + +func TestCatalogCache_ErrorWithoutPriorSuccess(t *testing.T) { + cache := newCatalogCache(time.Hour, func() ([]byte, error) { + return nil, errors.New("upstream down") + }) + + body, stale, err := cache.get() + if err == nil { + t.Fatal("get() with no prior success: expected error, got nil") + } + if stale { + t.Fatal("get() with no prior success: expected stale=false") + } + if body != nil { + t.Fatalf("get() with no prior success: body = %q, want nil", body) + } +} + +// --------------------------------------------------------------------------- +// fetchRemoteIndex +// --------------------------------------------------------------------------- + +func TestFetchRemoteIndex_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Accept"); got != "application/json" { + t.Errorf("Accept header = %q, want application/json", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"extensions":[]}`)) + })) + defer srv.Close() + + body, err := fetchRemoteIndex(srv.URL, srv.Client()) + if err != nil { + t.Fatalf("fetchRemoteIndex: %v", err) + } + if string(body) != `{"extensions":[]}` { + t.Fatalf("body = %q, want catalog JSON", body) + } +} + +func TestFetchRemoteIndex_NonOKStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + if _, err := fetchRemoteIndex(srv.URL, srv.Client()); err == nil { + t.Fatal("fetchRemoteIndex: expected error on 503, got nil") + } +} + +// --------------------------------------------------------------------------- +// fetchLocalIndex — the air-gapped path +// --------------------------------------------------------------------------- + +func TestFetchLocalIndex_Success(t *testing.T) { + path := filepath.Join(t.TempDir(), "index.json") + want := `{"extensions":[{"name":"plugin-hub"}]}` + if err := os.WriteFile(path, []byte(want), 0o644); err != nil { + t.Fatalf("seed file: %v", err) + } + + got, err := fetchLocalIndex(path) + if err != nil { + t.Fatalf("fetchLocalIndex: %v", err) + } + if string(got) != want { + t.Fatalf("body = %q, want %q", got, want) + } +} + +func TestFetchLocalIndex_MissingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "does-not-exist.json") + + _, err := fetchLocalIndex(path) + if err == nil { + t.Fatal("fetchLocalIndex: expected error for missing file, got nil") + } + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("fetchLocalIndex: error = %v, want it to wrap os.ErrNotExist", err) + } +} + +// --------------------------------------------------------------------------- +// hubIndexPath +// --------------------------------------------------------------------------- + +func TestHubIndexPath(t *testing.T) { + t.Run("unset", func(t *testing.T) { + t.Setenv("HUB_INDEX_PATH", "") + if got := hubIndexPath(); got != "" { + t.Fatalf("hubIndexPath() = %q, want empty", got) + } + }) + + t.Run("set", func(t *testing.T) { + t.Setenv("HUB_INDEX_PATH", "/etc/plugin-hub-index/index.json") + if got := hubIndexPath(); got != "/etc/plugin-hub-index/index.json" { + t.Fatalf("hubIndexPath() = %q, want the configured path", got) + } + }) +} diff --git a/backend/main.go b/backend/main.go index aac5b1a..31fa17c 100644 --- a/backend/main.go +++ b/backend/main.go @@ -3,10 +3,12 @@ // Surfaces three categories of HTTP routes: // // 1. Bundle + icon + health — served straight from embedded files. -// 2. Catalog — fetches the upstream hub index, caches it in +// 2. Catalog — fetches the hub index (from GitHub, or from +// a locally mounted file for air-gapped +// clusters — see HUB_INDEX_PATH), caches it in // memory, and returns the parsed JSON. Falls -// back to the last successful response when the -// upstream is unreachable. +// back to the last successful response when +// the source is unreachable. // 3. Installed extensions — proxied call to the OpenEverest API server, // forwarding the caller's X-Everest-User JWT. // 4. Summary — server-side join of catalog + installed, @@ -66,6 +68,15 @@ func hubIndexURL() string { return defaultHubIndexURL } +// hubIndexPath, when set, points at a local file holding the catalog index — +// typically a ConfigMap mounted into the pod (see the chart's localIndex +// values). It takes precedence over hubIndexURL: an air-gapped cluster can +// point this at the mounted file and the backend never has to reach GitHub +// for the catalog. Empty (the default) keeps the existing network fetch. +func hubIndexPath() string { + return os.Getenv("HUB_INDEX_PATH") +} + func cacheTTL() time.Duration { if v := os.Getenv("CACHE_TTL_SECONDS"); v != "" { if n, err := strconv.Atoi(v); err == nil && n > 0 { @@ -112,21 +123,23 @@ func clusterName() string { // --------------------------------------------------------------------------- type catalogCache struct { - mu sync.RWMutex - ttl time.Duration - url string - client *http.Client - body []byte // last successful raw JSON body - fetchedAt time.Time - lastErr error - lastErrAt time.Time -} - -func newCatalogCache(url string, ttl time.Duration) *catalogCache { + mu sync.RWMutex + ttl time.Duration + fetch func() ([]byte, error) // source-specific: HTTP GET or local file read + body []byte // last successful raw JSON body + fetchedAt time.Time + lastErr error + lastErrAt time.Time +} + +// newCatalogCache wraps fetch — fetchRemoteIndex for the default GitHub-hosted +// hub, or fetchLocalIndex for the air-gapped path — with TTL caching and +// stale-on-error fallback. The cache itself doesn't know or care which source +// it's given; main picks one, once, at startup, based on hubIndexPath. +func newCatalogCache(ttl time.Duration, fetch func() ([]byte, error)) *catalogCache { return &catalogCache{ - ttl: ttl, - url: url, - client: &http.Client{Timeout: upstreamFetchTimeout}, + ttl: ttl, + fetch: fetch, } } @@ -149,7 +162,7 @@ func (c *catalogCache) get() (body []byte, stale bool, err error) { return c.body, false, nil } - fresh, fetchErr := c.fetchLocked() + fresh, fetchErr := c.fetch() if fetchErr == nil { c.body = fresh c.fetchedAt = time.Now() @@ -166,13 +179,15 @@ func (c *catalogCache) get() (body []byte, stale bool, err error) { return nil, false, fetchErr } -func (c *catalogCache) fetchLocked() ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, c.url, nil) +// fetchRemoteIndex GETs the catalog index over HTTP — the GitHub-hosted hub +// by default, or whatever HUB_INDEX_URL overrides it to. +func fetchRemoteIndex(url string, client *http.Client) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, err } req.Header.Set("Accept", "application/json") - resp, err := c.client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, err } @@ -183,6 +198,19 @@ func (c *catalogCache) fetchLocked() ([]byte, error) { return io.ReadAll(resp.Body) } +// fetchLocalIndex reads the catalog index off disk — the air-gapped path, +// meant for a file mounted from a ConfigMap. A missing or not-yet-mounted +// volume is a normal, expected failure mode here, so it's reported through +// the same stale-on-error fallback get() already uses for a failed HTTP +// fetch, not a startup crash. +func fetchLocalIndex(path string) ([]byte, error) { + body, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read local hub index %q: %w", path, err) + } + return body, nil +} + // --------------------------------------------------------------------------- // Icon proxy // --------------------------------------------------------------------------- @@ -722,7 +750,7 @@ type summaryResponse struct { // --------------------------------------------------------------------------- func main() { - cache := newCatalogCache(hubIndexURL(), cacheTTL()) + cache, hubSource := newCatalogCacheFromEnv() icons := newIconProxy() mux := http.NewServeMux() @@ -736,8 +764,28 @@ func main() { port := listenPort() log.Printf("plugin-hub backend listening on :%s (hub: %s, everest: %s, cache TTL: %s)", - port, hubIndexURL(), everestAPIURL(), cacheTTL()) + port, hubSource, everestAPIURL(), cacheTTL()) if err := http.ListenAndServe(":"+port, mux); err != nil { log.Fatalf("server error: %v", err) } } + +// newCatalogCacheFromEnv picks the catalog source — local file if +// HUB_INDEX_PATH is set, otherwise the GitHub-hosted hub — and returns a +// ready-to-use cache plus a human-readable description of the source for the +// startup log. +func newCatalogCacheFromEnv() (*catalogCache, string) { + if path := hubIndexPath(); path != "" { + cache := newCatalogCache(cacheTTL(), func() ([]byte, error) { + return fetchLocalIndex(path) + }) + return cache, "file://" + path + " (air-gapped)" + } + + url := hubIndexURL() + client := &http.Client{Timeout: upstreamFetchTimeout} + cache := newCatalogCache(cacheTTL(), func() ([]byte, error) { + return fetchRemoteIndex(url, client) + }) + return cache, url +} diff --git a/charts/plugin-hub/templates/deployment.yaml b/charts/plugin-hub/templates/deployment.yaml index 14da937..a01e8ba 100644 --- a/charts/plugin-hub/templates/deployment.yaml +++ b/charts/plugin-hub/templates/deployment.yaml @@ -37,6 +37,10 @@ spec: - name: HUB_INDEX_URL value: {{ .Values.hubIndexURL | quote }} {{- end }} + {{- if .Values.localIndex.configMapName }} + - name: HUB_INDEX_PATH + value: "/etc/plugin-hub-index/{{ .Values.localIndex.key }}" + {{- end }} {{- if .Values.cacheTTLSeconds }} - name: CACHE_TTL_SECONDS value: {{ .Values.cacheTTLSeconds | quote }} @@ -45,6 +49,12 @@ spec: - name: EVEREST_CLUSTER_NAME value: {{ .Values.clusterName | quote }} {{- end }} + {{- if .Values.localIndex.configMapName }} + volumeMounts: + - name: local-index + mountPath: /etc/plugin-hub-index + readOnly: true + {{- end }} livenessProbe: httpGet: path: /healthz @@ -59,3 +69,9 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.localIndex.configMapName }} + volumes: + - name: local-index + configMap: + name: {{ .Values.localIndex.configMapName }} + {{- end }} diff --git a/charts/plugin-hub/values.yaml b/charts/plugin-hub/values.yaml index 763466a..a59e2ed 100644 --- a/charts/plugin-hub/values.yaml +++ b/charts/plugin-hub/values.yaml @@ -38,6 +38,15 @@ everestAPIURL: "" # Override the catalog index URL. Defaults to the official OpenEverest hub. hubIndexURL: "" +# Air-gapped support: mount a pre-populated catalog index from an existing +# ConfigMap instead of fetching hubIndexURL from GitHub. Leave configMapName +# empty to keep the default network fetch. The ConfigMap must contain a key +# (see `key` below) holding the same JSON shape as the upstream hub index. +# When set, this takes precedence over hubIndexURL. +localIndex: + configMapName: "" + key: index.json + # Catalog cache TTL in seconds. Defaults to 300 (5 minutes) when empty. cacheTTLSeconds: ""