Support air-gapped catalogs via a local ConfigMap-mounted index

README.md

@@ -5,7 +5,8 @@ The Hub plugin for [OpenEverest](https://github.com/openeverest/openeverest). It
 - **Sidebar entry + dedicated route** registered as an OpenEverest generic plugin.
 - **Searchable, filterable table** of all catalog entries (plugins and providers) with install status, default-channel version, categories, and a per-entry detail drawer.
 - **Backend-side join**: a single `/api/summary` call merges the hub catalog with the live `InstalledExtension` list so the UI does one round-trip.
-- **Stale-tolerant**: the upstream catalog is cached in-memory; if GitHub becomes unreachable the last successful response is served with an `X-Hub-Stale: true` header.
+- **Stale-tolerant**: the catalog is cached in-memory; if the source becomes unreachable the last successful response is served with an `X-Hub-Stale: true` header.
+- **Air-gapped friendly**: point `localIndex.configMapName` (chart) or `HUB_INDEX_PATH` (backend) at a locally-provided `index.json` and the backend never has to reach GitHub for the catalog.
 - **Browse-only in v1.** Each catalog entry shows a copy-pasteable `helm install …` command. Install/uninstall actions from the UI will land in a later release.
 
 Built from the [`openeverest/generic-plugin-template`](https://github.com/openeverest/generic-plugin-template) and following the [Generic Plugins Architecture Design](https://github.com/openeverest/openeverest/blob/main/docs/process/generic-plugins-design.md).
@@ -47,8 +48,27 @@ Edit `charts/plugin-hub/values.yaml` (or pass `--set` flags):
 | `plugin.enabled` | Enable/disable the plugin | `true` |
 | `everestAPIURL` | OpenEverest API server URL (defaults to in-cluster DNS `everest.everest-system.svc.cluster.local:8080` when empty) | `""` |
 | `hubIndexURL` | Override the catalog index URL | official hub when empty |
+| `localIndex.configMapName` | Existing ConfigMap holding a local catalog index, for air-gapped clusters. Mounted at `/etc/plugin-hub-index` and takes precedence over `hubIndexURL` when set. | `""` |
+| `localIndex.key` | Key inside that ConfigMap holding the index JSON | `index.json` |
 | `cacheTTLSeconds` | Catalog cache TTL (seconds) | `300` when empty |
 
+## Air-gapped / offline catalog
+
+For clusters without outbound access to GitHub, provide the catalog index from a ConfigMap instead:
+
+```bash
+kubectl create configmap plugin-hub-index \
+  --from-file=index.json=./index.json \
+  -n everest-system
+
+helm upgrade --install plugin-hub oci://ghcr.io/openeverest/charts/plugin-hub \
+  --version 0.1.14 \
+  -n everest-system \
+  --set localIndex.configMapName=plugin-hub-index
+```
+
+The chart mounts the ConfigMap as a directory (not a `subPath` file), so the backend picks up updates on its normal cache TTL after `kubectl apply`-ing a new ConfigMap revision — no pod restart needed, the same way the GitHub-backed catalog already refreshes on TTL expiry.
+
 ## Local development
 
 ```bash
@@ -105,6 +125,9 @@ docker build -t plugin-hub:dev .
             │                  │       ▼           │
             │                  │   GitHub raw      │
             │                  │   hub/index.json  │
+            │                  │   — or a mounted  │
+            │                  │   ConfigMap file  │
+            │                  │   (air-gapped)    │
             │                  │                   │
             │                  └─► OpenEverest API │
             │                      /v1/installed-  │

backend/catalog_test.go

@@ -0,0 +1,188 @@
+package main
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// ---------------------------------------------------------------------------
+// catalogCache
+// ---------------------------------------------------------------------------
+
+func TestCatalogCache_ServesFreshWithinTTL(t *testing.T) {
+	calls := 0
+	cache := newCatalogCache(time.Hour, func() ([]byte, error) {
+		calls++
+		return []byte("body"), nil
+	})
+
+	for i := 0; i < 3; i++ {
+		body, stale, err := cache.get()
+		if err != nil {
+			t.Fatalf("get() #%d: unexpected error: %v", i, err)
+		}
+		if stale {
+			t.Fatalf("get() #%d: expected fresh, got stale", i)
+		}
+		if string(body) != "body" {
+			t.Fatalf("get() #%d: body = %q, want %q", i, body, "body")
+		}
+	}
+	if calls != 1 {
+		t.Fatalf("fetch called %d times within TTL, want 1", calls)
+	}
+}
+
+func TestCatalogCache_RefetchesAfterTTL(t *testing.T) {
+	calls := 0
+	cache := newCatalogCache(5*time.Millisecond, func() ([]byte, error) {
+		calls++
+		return []byte("body"), nil
+	})
+
+	if _, _, err := cache.get(); err != nil {
+		t.Fatalf("first get(): %v", err)
+	}
+	time.Sleep(15 * time.Millisecond)
+	if _, _, err := cache.get(); err != nil {
+		t.Fatalf("second get(): %v", err)
+	}
+	if calls != 2 {
+		t.Fatalf("fetch called %d times across TTL expiry, want 2", calls)
+	}
+}
+
+func TestCatalogCache_ServesStaleOnFetchError(t *testing.T) {
+	calls := 0
+	cache := newCatalogCache(5*time.Millisecond, func() ([]byte, error) {
+		calls++
+		if calls == 1 {
+			return []byte("good"), nil
+		}
+		return nil, errors.New("upstream down")
+	})
+
+	if _, _, err := cache.get(); err != nil {
+		t.Fatalf("first get(): %v", err)
+	}
+	time.Sleep(15 * time.Millisecond)
+
+	body, stale, err := cache.get()
+	if err != nil {
+		t.Fatalf("get() after fetch failure: unexpected error: %v", err)
+	}
+	if !stale {
+		t.Fatal("get() after fetch failure: expected stale=true")
+	}
+	if string(body) != "good" {
+		t.Fatalf("get() after fetch failure: body = %q, want last-good %q", body, "good")
+	}
+}
+
+func TestCatalogCache_ErrorWithoutPriorSuccess(t *testing.T) {
+	cache := newCatalogCache(time.Hour, func() ([]byte, error) {
+		return nil, errors.New("upstream down")
+	})
+
+	body, stale, err := cache.get()
+	if err == nil {
+		t.Fatal("get() with no prior success: expected error, got nil")
+	}
+	if stale {
+		t.Fatal("get() with no prior success: expected stale=false")
+	}
+	if body != nil {
+		t.Fatalf("get() with no prior success: body = %q, want nil", body)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// fetchRemoteIndex
+// ---------------------------------------------------------------------------
+
+func TestFetchRemoteIndex_Success(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if got := r.Header.Get("Accept"); got != "application/json" {
+			t.Errorf("Accept header = %q, want application/json", got)
+		}
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"extensions":[]}`))
+	}))
+	defer srv.Close()
+
+	body, err := fetchRemoteIndex(srv.URL, srv.Client())
+	if err != nil {
+		t.Fatalf("fetchRemoteIndex: %v", err)
+	}
+	if string(body) != `{"extensions":[]}` {
+		t.Fatalf("body = %q, want catalog JSON", body)
+	}
+}
+
+func TestFetchRemoteIndex_NonOKStatus(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer srv.Close()
+
+	if _, err := fetchRemoteIndex(srv.URL, srv.Client()); err == nil {
+		t.Fatal("fetchRemoteIndex: expected error on 503, got nil")
+	}
+}
+
+// ---------------------------------------------------------------------------
+// fetchLocalIndex — the air-gapped path
+// ---------------------------------------------------------------------------
+
+func TestFetchLocalIndex_Success(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "index.json")
+	want := `{"extensions":[{"name":"plugin-hub"}]}`
+	if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
+		t.Fatalf("seed file: %v", err)
+	}
+
+	got, err := fetchLocalIndex(path)
+	if err != nil {
+		t.Fatalf("fetchLocalIndex: %v", err)
+	}
+	if string(got) != want {
+		t.Fatalf("body = %q, want %q", got, want)
+	}
+}
+
+func TestFetchLocalIndex_MissingFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "does-not-exist.json")
+
+	_, err := fetchLocalIndex(path)
+	if err == nil {
+		t.Fatal("fetchLocalIndex: expected error for missing file, got nil")
+	}
+	if !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("fetchLocalIndex: error = %v, want it to wrap os.ErrNotExist", err)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// hubIndexPath
+// ---------------------------------------------------------------------------
+
+func TestHubIndexPath(t *testing.T) {
+	t.Run("unset", func(t *testing.T) {
+		t.Setenv("HUB_INDEX_PATH", "")
+		if got := hubIndexPath(); got != "" {
+			t.Fatalf("hubIndexPath() = %q, want empty", got)
+		}
+	})
+
+	t.Run("set", func(t *testing.T) {
+		t.Setenv("HUB_INDEX_PATH", "/etc/plugin-hub-index/index.json")
+		if got := hubIndexPath(); got != "/etc/plugin-hub-index/index.json" {
+			t.Fatalf("hubIndexPath() = %q, want the configured path", got)
+		}
+	})
+}

backend/main.go

@@ -3,10 +3,12 @@
 // Surfaces three categories of HTTP routes:
 //
 //  1. Bundle + icon + health   — served straight from embedded files.
-//  2. Catalog                  — fetches the upstream hub index, caches it in
+//  2. Catalog                  — fetches the hub index (from GitHub, or from
+//                                a locally mounted file for air-gapped
+//                                clusters — see HUB_INDEX_PATH), caches it in
 //                                memory, and returns the parsed JSON. Falls
-//                                back to the last successful response when the
-//                                upstream is unreachable.
+//                                back to the last successful response when
+//                                the source is unreachable.
 //  3. Installed extensions     — proxied call to the OpenEverest API server,
 //                                forwarding the caller's X-Everest-User JWT.
 //  4. Summary                  — server-side join of catalog + installed,
@@ -66,6 +68,15 @@ func hubIndexURL() string {
 	return defaultHubIndexURL
 }
 
+// hubIndexPath, when set, points at a local file holding the catalog index —
+// typically a ConfigMap mounted into the pod (see the chart's localIndex
+// values). It takes precedence over hubIndexURL: an air-gapped cluster can
+// point this at the mounted file and the backend never has to reach GitHub
+// for the catalog. Empty (the default) keeps the existing network fetch.
+func hubIndexPath() string {
+	return os.Getenv("HUB_INDEX_PATH")
+}
+
 func cacheTTL() time.Duration {
 	if v := os.Getenv("CACHE_TTL_SECONDS"); v != "" {
 		if n, err := strconv.Atoi(v); err == nil && n > 0 {
@@ -112,21 +123,23 @@ func clusterName() string {
 // ---------------------------------------------------------------------------
 
 type catalogCache struct {
-	mu          sync.RWMutex
-	ttl         time.Duration
-	url         string
-	client      *http.Client
-	body        []byte // last successful raw JSON body
-	fetchedAt   time.Time
-	lastErr     error
-	lastErrAt   time.Time
-}
-
-func newCatalogCache(url string, ttl time.Duration) *catalogCache {
+	mu        sync.RWMutex
+	ttl       time.Duration
+	fetch     func() ([]byte, error) // source-specific: HTTP GET or local file read
+	body      []byte                 // last successful raw JSON body
+	fetchedAt time.Time
+	lastErr   error
+	lastErrAt time.Time
+}
+
+// newCatalogCache wraps fetch — fetchRemoteIndex for the default GitHub-hosted
+// hub, or fetchLocalIndex for the air-gapped path — with TTL caching and
+// stale-on-error fallback. The cache itself doesn't know or care which source
+// it's given; main picks one, once, at startup, based on hubIndexPath.
+func newCatalogCache(ttl time.Duration, fetch func() ([]byte, error)) *catalogCache {
 	return &catalogCache{
-		ttl:    ttl,
-		url:    url,
-		client: &http.Client{Timeout: upstreamFetchTimeout},
+		ttl:   ttl,
+		fetch: fetch,
 	}
 }
 
@@ -149,7 +162,7 @@ func (c *catalogCache) get() (body []byte, stale bool, err error) {
 		return c.body, false, nil
 	}
 
-	fresh, fetchErr := c.fetchLocked()
+	fresh, fetchErr := c.fetch()
 	if fetchErr == nil {
 		c.body = fresh
 		c.fetchedAt = time.Now()
@@ -166,13 +179,15 @@ func (c *catalogCache) get() (body []byte, stale bool, err error) {
 	return nil, false, fetchErr
 }
 
-func (c *catalogCache) fetchLocked() ([]byte, error) {
-	req, err := http.NewRequest(http.MethodGet, c.url, nil)
+// fetchRemoteIndex GETs the catalog index over HTTP — the GitHub-hosted hub
+// by default, or whatever HUB_INDEX_URL overrides it to.
+func fetchRemoteIndex(url string, client *http.Client) ([]byte, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Set("Accept", "application/json")
-	resp, err := c.client.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -183,6 +198,19 @@ func (c *catalogCache) fetchLocked() ([]byte, error) {
 	return io.ReadAll(resp.Body)
 }
 
+// fetchLocalIndex reads the catalog index off disk — the air-gapped path,
+// meant for a file mounted from a ConfigMap. A missing or not-yet-mounted
+// volume is a normal, expected failure mode here, so it's reported through
+// the same stale-on-error fallback get() already uses for a failed HTTP
+// fetch, not a startup crash.
+func fetchLocalIndex(path string) ([]byte, error) {
+	body, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read local hub index %q: %w", path, err)
+	}
+	return body, nil
+}
+
 // ---------------------------------------------------------------------------
 // Icon proxy
 // ---------------------------------------------------------------------------
@@ -722,7 +750,7 @@ type summaryResponse struct {
 // ---------------------------------------------------------------------------
 
 func main() {
-	cache := newCatalogCache(hubIndexURL(), cacheTTL())
+	cache, hubSource := newCatalogCacheFromEnv()
 	icons := newIconProxy()
 
 	mux := http.NewServeMux()
@@ -736,8 +764,28 @@ func main() {
 
 	port := listenPort()
 	log.Printf("plugin-hub backend listening on :%s (hub: %s, everest: %s, cache TTL: %s)",
-		port, hubIndexURL(), everestAPIURL(), cacheTTL())
+		port, hubSource, everestAPIURL(), cacheTTL())
 	if err := http.ListenAndServe(":"+port, mux); err != nil {
 		log.Fatalf("server error: %v", err)
 	}
 }
+
+// newCatalogCacheFromEnv picks the catalog source — local file if
+// HUB_INDEX_PATH is set, otherwise the GitHub-hosted hub — and returns a
+// ready-to-use cache plus a human-readable description of the source for the
+// startup log.
+func newCatalogCacheFromEnv() (*catalogCache, string) {
+	if path := hubIndexPath(); path != "" {
+		cache := newCatalogCache(cacheTTL(), func() ([]byte, error) {
+			return fetchLocalIndex(path)
+		})
+		return cache, "file://" + path + " (air-gapped)"
+	}
+
+	url := hubIndexURL()
+	client := &http.Client{Timeout: upstreamFetchTimeout}
+	cache := newCatalogCache(cacheTTL(), func() ([]byte, error) {
+		return fetchRemoteIndex(url, client)
+	})
+	return cache, url
+}