From 182029a9752ea18b3aaee87acca8b29768bc5e17 Mon Sep 17 00:00:00 2001 From: Nicolas Bigler Date: Wed, 22 Apr 2026 09:56:01 +0000 Subject: [PATCH] Add SLI prober for Garage --- apis/vshn/v1/dbaas_vshn_garage.go | 10 + apis/vshn/v1/zz_generated.deepcopy.go | 1 + cmd/sliexporter.go | 15 + config/controller/webhooks.yaml | 1 - config/sliexporter/rbac/role.yaml | 15 + crds/vshn.appcat.vshn.io_xvshngarages.yaml | 9 + .../functions/vshngarage/deploy.go | 3 + pkg/sliexporter/probes/garage.go | 127 +++++++ pkg/sliexporter/probes/garage_test.go | 135 ++++++++ .../vshngarage_controller.go | 264 +++++++++++++++ .../vshngarage_controller_test.go | 317 ++++++++++++++++++ 11 files changed, 896 insertions(+), 1 deletion(-) create mode 100644 pkg/sliexporter/probes/garage.go create mode 100644 pkg/sliexporter/probes/garage_test.go create mode 100644 pkg/sliexporter/vshngarage_controller/vshngarage_controller.go create mode 100644 pkg/sliexporter/vshngarage_controller/vshngarage_controller_test.go diff --git a/apis/vshn/v1/dbaas_vshn_garage.go b/apis/vshn/v1/dbaas_vshn_garage.go index 08639492d1..778baabda0 100644 --- a/apis/vshn/v1/dbaas_vshn_garage.go +++ b/apis/vshn/v1/dbaas_vshn_garage.go @@ -4,6 +4,7 @@ import ( "fmt" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + cpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -131,6 +132,7 @@ type XVSHNGarageSpec struct { // Parameters are the configurable fields of a VSHNGarage. Parameters VSHNGarageParameters `json:"parameters,omitempty"` + CompositionRef cpv1.CompositionReference `json:"compositionRef,omitempty"` xpv1.ResourceSpec `json:",inline"` } @@ -214,6 +216,14 @@ func (v *VSHNGarage) GetBackupRetention() K8upRetentionPolicy { return K8upRetentionPolicy{} } +func (v *XVSHNGarage) GetInstanceNamespace() string { + return fmt.Sprintf("vshn-garage-%s", v.GetName()) +} + +func (v *XVSHNGarage) GetInstances() int { + return v.Spec.Parameters.Instances +} + // GetBackupSchedule returns the current backup schedule func (v *VSHNGarage) GetBackupSchedule() string { return v.Status.Schedules.Backup diff --git a/apis/vshn/v1/zz_generated.deepcopy.go b/apis/vshn/v1/zz_generated.deepcopy.go index 1bfacbe54d..cdced38b97 100644 --- a/apis/vshn/v1/zz_generated.deepcopy.go +++ b/apis/vshn/v1/zz_generated.deepcopy.go @@ -2461,6 +2461,7 @@ func (in *XVSHNGarageList) DeepCopyObject() runtime.Object { func (in *XVSHNGarageSpec) DeepCopyInto(out *XVSHNGarageSpec) { *out = *in in.Parameters.DeepCopyInto(&out.Parameters) + out.CompositionRef = in.CompositionRef in.ResourceSpec.DeepCopyInto(&out.ResourceSpec) } diff --git a/cmd/sliexporter.go b/cmd/sliexporter.go index 4d31e1eb64..3ed910326d 100644 --- a/cmd/sliexporter.go +++ b/cmd/sliexporter.go @@ -12,6 +12,7 @@ import ( "github.com/vshn/appcat/v4/pkg/common/utils" maintenancecontroller "github.com/vshn/appcat/v4/pkg/sliexporter/maintenance_controller" "github.com/vshn/appcat/v4/pkg/sliexporter/probes" + vshngaragecontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshngarage_controller" vshnkeycloakcontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnkeycloak_controller" vshnmariadbcontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnmariadb_controller" vshnminiocontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnminio_controller" @@ -177,6 +178,20 @@ func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error { return err } } + if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNGarage", ctrl.GetConfigOrDie()) { + log.Info("Enabling VSHNGarage controller") + if err = (&vshngaragecontroller.VSHNGarageReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ProbeManager: &probeManager, + StartupGracePeriod: startupGraceMin * time.Minute, + GarageDialer: probes.NewGarage, + ScClient: scClient, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "VSHNGarage") + return err + } + } if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNMariaDB", ctrl.GetConfigOrDie()) { log.Info("Enabling VSHNMariaDB controller") if err = (&vshnmariadbcontroller.VSHNMariaDBReconciler{ diff --git a/config/controller/webhooks.yaml b/config/controller/webhooks.yaml index 885a0620c3..c324049e1c 100644 --- a/config/controller/webhooks.yaml +++ b/config/controller/webhooks.yaml @@ -1,4 +1,3 @@ ---- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: diff --git a/config/sliexporter/rbac/role.yaml b/config/sliexporter/rbac/role.yaml index b163e4c43f..217f3fb4d8 100644 --- a/config/sliexporter/rbac/role.yaml +++ b/config/sliexporter/rbac/role.yaml @@ -13,6 +13,17 @@ rules: - get - list - watch +- apiGroups: + - garage.rajsingh.info + resources: + - garagebuckets + - garagekeys + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - managedupgrade.appuio.io resources: @@ -24,10 +35,12 @@ rules: - apiGroups: - vshn.appcat.vshn.io resources: + - vshngarages - vshnmariadbs - vshnminios - vshnpostgresqls - vshnredis + - xvshngarages - xvshnkeycloaks - xvshnmariadbs - xvshnminios @@ -40,10 +53,12 @@ rules: - apiGroups: - vshn.appcat.vshn.io resources: + - vshngarages/status - vshnmariadbs/status - vshnminios/status - vshnpostgresqls/status - vshnredis/status + - xvshngarages/status - xvshnkeycloaks/status - xvshnmariadbs/status - xvshnminios/status diff --git a/crds/vshn.appcat.vshn.io_xvshngarages.yaml b/crds/vshn.appcat.vshn.io_xvshngarages.yaml index c87620e31a..36f39d1737 100644 --- a/crds/vshn.appcat.vshn.io_xvshngarages.yaml +++ b/crds/vshn.appcat.vshn.io_xvshngarages.yaml @@ -39,6 +39,15 @@ spec: spec: description: XVSHNGarageSpec defines the desired state of a VSHNGarage. properties: + compositionRef: + description: A CompositionReference references a Composition. + properties: + name: + description: Name of the Composition. + type: string + required: + - name + type: object deletionPolicy: default: Delete description: |- diff --git a/pkg/comp-functions/functions/vshngarage/deploy.go b/pkg/comp-functions/functions/vshngarage/deploy.go index c01f2d9db3..ac3f0a09fc 100644 --- a/pkg/comp-functions/functions/vshngarage/deploy.go +++ b/pkg/comp-functions/functions/vshngarage/deploy.go @@ -63,6 +63,9 @@ func DeployGarage(ctx context.Context, comp *vshnv1.VSHNGarage, svc *runtime.Ser "memory": calcResources.Mem, }, }, + "adminKey": map[string]any{ + "enabled": true, + }, "storageDataSpace": calcResources.Disk.String(), "storageMetadataSpace": comp.Spec.Parameters.Service.MetadataStorage, } diff --git a/pkg/sliexporter/probes/garage.go b/pkg/sliexporter/probes/garage.go new file mode 100644 index 0000000000..8244e2889e --- /dev/null +++ b/pkg/sliexporter/probes/garage.go @@ -0,0 +1,127 @@ +package probes + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + miniolib "github.com/minio/minio-go/v7" +) + +type VSHNGarage struct { + minioClient *miniolib.Client + httpClient *http.Client + adminURL string + adminToken string + bucketName string + Service string + Name string + ClaimNamespace string + InstanceNamespace string + HighAvailable bool + Organization string + ServiceLevel string + CompositionName string +} + +func (g VSHNGarage) Close() error { + return nil +} + +func (g VSHNGarage) GetInfo() ProbeInfo { + return ProbeInfo{ + Service: g.Service, + Name: g.Name, + ClaimNamespace: g.ClaimNamespace, + InstanceNamespace: g.InstanceNamespace, + HighAvailable: g.HighAvailable, + Organization: g.Organization, + ServiceLevel: g.ServiceLevel, + CompositionName: g.CompositionName, + } +} + +func (g VSHNGarage) Probe(ctx context.Context) error { + if err := g.checkAdminHealth(ctx); err != nil { + return err + } + return g.checkS3Write(ctx) +} + +func (g VSHNGarage) checkAdminHealth(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.adminURL+"/v2/GetClusterHealth", nil) + if err != nil { + return fmt.Errorf("cannot create admin health request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+g.adminToken) + + resp, err := g.httpClient.Do(req) + if err != nil { + return fmt.Errorf("admin health check failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("admin API returned status %d: %s", resp.StatusCode, body) + } + + var health struct { + Status string `json:"status"` + } + if err := json.NewDecoder(resp.Body).Decode(&health); err != nil { + return fmt.Errorf("cannot decode admin health response: %w", err) + } + if health.Status != "healthy" { + return fmt.Errorf("garage cluster health is %q", health.Status) + } + return nil +} + +func (g VSHNGarage) checkS3Write(ctx context.Context) error { + x := bytes.NewBufferString("This file is auto-generated, do not edit, VSHN SLI Exporter purposes") + _, err := g.minioClient.PutObject(ctx, g.bucketName, "vshn-sli-probe", x, int64(x.Len()), miniolib.PutObjectOptions{ContentType: "application/octet-stream"}) + return err +} + +func NewGarage(service, name, claimNamespace, instanceNamespace, organization, sla, compositionName, endpointURL, adminToken, bucketName string, ha bool, opts miniolib.Options) (*VSHNGarage, error) { + u, err := url.Parse(endpointURL) + if err != nil { + return nil, fmt.Errorf("cannot parse garage endpoint %q: %w", endpointURL, err) + } + if u.Host == "" { + return nil, fmt.Errorf("garage endpoint %q has no host (add scheme, e.g. http://)", endpointURL) + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("garage endpoint %q has unsupported scheme %q (must be http or https)", endpointURL, u.Scheme) + } + + opts.Secure = u.Scheme == "https" + minioClient, err := miniolib.New(u.Host, &opts) + if err != nil { + return nil, err + } + + adminURL := fmt.Sprintf("%s://%s:3903", u.Scheme, u.Hostname()) + + return &VSHNGarage{ + minioClient: minioClient, + httpClient: &http.Client{Timeout: 5 * time.Second}, + adminURL: adminURL, + adminToken: adminToken, + bucketName: bucketName, + Service: service, + Name: name, + ClaimNamespace: claimNamespace, + InstanceNamespace: instanceNamespace, + HighAvailable: ha, + Organization: organization, + ServiceLevel: sla, + CompositionName: compositionName, + }, nil +} diff --git a/pkg/sliexporter/probes/garage_test.go b/pkg/sliexporter/probes/garage_test.go new file mode 100644 index 0000000000..7c6439e38f --- /dev/null +++ b/pkg/sliexporter/probes/garage_test.go @@ -0,0 +1,135 @@ +package probes + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + miniolib "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// garageWithAdmin builds a VSHNGarage whose admin URL points to the given test +// server and whose S3 client points to s3Host. Keeps test setup minimal. +func garageWithAdmin(adminServer *httptest.Server, s3Host string) VSHNGarage { + mc, _ := miniolib.New(s3Host, &miniolib.Options{ + Creds: credentials.NewStaticV4("key", "secret", ""), + }) + return VSHNGarage{ + httpClient: &http.Client{}, + adminURL: adminServer.URL, + adminToken: "test-token", + bucketName: "test-bucket", + minioClient: mc, + } +} + +// TestNewGarage_EndpointParsing covers URL validation and admin URL derivation. + +func TestNewGarage_MissingScheme(t *testing.T) { + opts := miniolib.Options{Creds: credentials.NewStaticV4("k", "s", "")} + _, err := NewGarage("svc", "n", "ns", "ins", "org", "be", "comp", "s3.example.com:3900", "tok", "bkt", false, opts) + assert.ErrorContains(t, err, "has no host") +} + +func TestNewGarage_EmptyEndpoint(t *testing.T) { + opts := miniolib.Options{Creds: credentials.NewStaticV4("k", "s", "")} + _, err := NewGarage("svc", "n", "ns", "ins", "org", "be", "comp", "", "tok", "bkt", false, opts) + assert.ErrorContains(t, err, "has no host") +} + +func TestNewGarage_MissingSchemeWithHost(t *testing.T) { + opts := miniolib.Options{Creds: credentials.NewStaticV4("k", "s", "")} + _, err := NewGarage("svc", "n", "ns", "ins", "org", "be", "comp", "//s3.example.com:3900", "tok", "bkt", false, opts) + assert.ErrorContains(t, err, "unsupported scheme") +} + +func TestNewGarage_UnsupportedScheme(t *testing.T) { + opts := miniolib.Options{Creds: credentials.NewStaticV4("k", "s", "")} + _, err := NewGarage("svc", "n", "ns", "ins", "org", "be", "comp", "s3://bucket.example.com", "tok", "bkt", false, opts) + assert.ErrorContains(t, err, "unsupported scheme") +} + +func TestNewGarage_AdminURLDerivedFromEndpoint(t *testing.T) { + opts := miniolib.Options{Creds: credentials.NewStaticV4("k", "s", "")} + g, err := NewGarage("svc", "n", "ns", "ins", "org", "be", "comp", "http://s3.example.com:3900", "tok", "bkt", false, opts) + require.NoError(t, err) + assert.Equal(t, "http://s3.example.com:3903", g.adminURL) +} + +func TestNewGarage_HTTPSEndpoint(t *testing.T) { + opts := miniolib.Options{Creds: credentials.NewStaticV4("k", "s", "")} + g, err := NewGarage("svc", "n", "ns", "ins", "org", "be", "comp", "https://s3.example.com:3900", "tok", "bkt", false, opts) + require.NoError(t, err) + assert.Equal(t, "https://s3.example.com:3903", g.adminURL) +} + +// TestGarage_Probe_* covers Probe error paths. + +func TestGarage_Probe_AdminNon200(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + g := garageWithAdmin(server, "localhost:1") + err := g.Probe(context.Background()) + assert.ErrorContains(t, err, "503") +} + +func TestGarage_Probe_AdminUnhealthy(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"status": "degraded"}) + })) + defer server.Close() + + g := garageWithAdmin(server, "localhost:1") + err := g.Probe(context.Background()) + assert.ErrorContains(t, err, `"degraded"`) +} + +func TestGarage_Probe_AdminInvalidJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not json")) + })) + defer server.Close() + + g := garageWithAdmin(server, "localhost:1") + err := g.Probe(context.Background()) + assert.ErrorContains(t, err, "cannot decode admin health response") +} + +func TestGarage_Probe_S3WriteFails(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"status": "healthy"}) + })) + defer server.Close() + + // Admin health passes; S3 client targets a port with nothing listening. + g := garageWithAdmin(server, "localhost:1") + err := g.Probe(context.Background()) + assert.Error(t, err) +} + +func TestGarage_Probe_ContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + mc, _ := miniolib.New("localhost:1", &miniolib.Options{ + Creds: credentials.NewStaticV4("k", "s", ""), + }) + g := VSHNGarage{ + httpClient: &http.Client{}, + adminURL: "http://localhost:1", + adminToken: "token", + bucketName: "test-bucket", + minioClient: mc, + } + + err := g.Probe(ctx) + assert.ErrorIs(t, err, context.Canceled) +} diff --git a/pkg/sliexporter/vshngarage_controller/vshngarage_controller.go b/pkg/sliexporter/vshngarage_controller/vshngarage_controller.go new file mode 100644 index 0000000000..519ff11a7f --- /dev/null +++ b/pkg/sliexporter/vshngarage_controller/vshngarage_controller.go @@ -0,0 +1,264 @@ +package vshngaragecontroller + +import ( + "context" + "crypto/sha256" + "fmt" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1" + "github.com/vshn/appcat/v4/pkg/common/utils" + "github.com/vshn/appcat/v4/pkg/sliexporter/probes" + slireconciler "github.com/vshn/appcat/v4/pkg/sliexporter/sli_reconciler" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + vshnGarageServiceKey = "VSHNGarage" + adminTokenSecret = "garage-admin-token" +) + +var garageBucketGVK = schema.GroupVersionKind{ + Group: "garage.rajsingh.info", + Version: "v1alpha1", + Kind: "GarageBucket", +} + +var garageKeyGVK = schema.GroupVersionKind{ + Group: "garage.rajsingh.info", + Version: "v1alpha1", + Kind: "GarageKey", +} + +type VSHNGarageReconciler struct { + client.Client + Scheme *runtime.Scheme + + ProbeManager probeManager + StartupGracePeriod time.Duration + GarageDialer func(service, name, claimNamespace, instanceNamespace, organization, sla, compositionName, endpointURL, adminToken, bucketName string, ha bool, opts minio.Options) (*probes.VSHNGarage, error) + ScClient client.Client +} + +type probeManager interface { + slireconciler.ProbeManager +} + +//+kubebuilder:rbac:groups=vshn.appcat.vshn.io,resources=xvshngarages,verbs=get;list;watch +//+kubebuilder:rbac:groups=vshn.appcat.vshn.io,resources=xvshngarages/status,verbs=get +//+kubebuilder:rbac:groups=vshn.appcat.vshn.io,resources=vshngarages,verbs=get;list;watch +//+kubebuilder:rbac:groups=vshn.appcat.vshn.io,resources=vshngarages/status,verbs=get + +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch + +//+kubebuilder:rbac:groups=garage.rajsingh.info,resources=garagebuckets,verbs=get;list;watch;create;delete +//+kubebuilder:rbac:groups=garage.rajsingh.info,resources=garagekeys,verbs=get;list;watch;create;delete + +func (r *VSHNGarageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + l := log.FromContext(ctx).WithValues("namespace", req.Namespace, "instance", req.Name) + l.Info("Reconciling XVSHNGarage") + + inst := &vshnv1.XVSHNGarage{} + if err := r.Get(ctx, req.NamespacedName, inst); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if inst.GetDeletionTimestamp() != nil { + if err := r.cleanupSLIBucketCR(ctx, inst); err != nil { + return ctrl.Result{}, err + } + } + + reconciler := slireconciler.New(inst, l, r.ProbeManager, vshnGarageServiceKey, req.NamespacedName, r.Client, r.StartupGracePeriod, r.getGarageProber, r.ScClient) + return reconciler.Reconcile(ctx) +} + +func (r VSHNGarageReconciler) getGarageProber(ctx context.Context, obj slireconciler.Service) (probes.Prober, error) { + inst, ok := obj.(*vshnv1.XVSHNGarage) + if !ok { + return nil, fmt.Errorf("cannot start probe, object not a valid XVSHNGarage") + } + + l := log.FromContext(ctx).WithValues( + "namespace", inst.Labels[slireconciler.ClaimNamespaceLabel], + "instance", inst.Labels[slireconciler.ClaimNameLabel], + ) + + instanceNamespace := inst.GetInstanceNamespace() + if instanceNamespace == "" { + return nil, fmt.Errorf("instance namespace not yet set for %s", inst.Name) + } + + claimNamespace := inst.Labels[slireconciler.ClaimNamespaceLabel] + claimName := inst.Labels[slireconciler.ClaimNameLabel] + + if err := r.ensureGarageBucketCR(ctx, claimNamespace, claimName, instanceNamespace); err != nil { + return nil, fmt.Errorf("cannot ensure SLI bucket CR: %w", err) + } + if err := r.ensureGarageKeyCR(ctx, claimNamespace, claimName, instanceNamespace); err != nil { + return nil, fmt.Errorf("cannot ensure SLI key CR: %w", err) + } + + bucketName := sliGarageBucketCRName(claimName) + l.Info("looking for bucket credentials secret", "name", bucketName, "namespace", claimNamespace) + bucketSecret := corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: bucketName, Namespace: claimNamespace}, &bucketSecret); err != nil { + return nil, err + } + + l.Info("looking for secret: "+adminTokenSecret, "namespace", instanceNamespace) + tokenSecret := corev1.Secret{} + if err := r.ScClient.Get(ctx, types.NamespacedName{Name: adminTokenSecret, Namespace: instanceNamespace}, &tokenSecret); err != nil { + return nil, err + } + + ha := inst.Spec.Parameters.Instances > 1 + sla := inst.Spec.Parameters.Service.ServiceLevel + compositionName := inst.Spec.CompositionRef.Name + + prober, err := r.GarageDialer( + vshnGarageServiceKey, + inst.Name, + claimNamespace, + instanceNamespace, + inst.GetLabels()[utils.OrgLabelName], + string(sla), + compositionName, + string(bucketSecret.Data["endpoint"]), + string(tokenSecret.Data["token"]), + bucketName, + ha, + minio.Options{ + Creds: credentials.NewStaticV4( + string(bucketSecret.Data["access-key-id"]), + string(bucketSecret.Data["secret-access-key"]), + "", + ), + }, + ) + if err != nil { + l.Error(err, "Can't create Garage prober") + return nil, err + } + return prober, nil +} + +func (r *VSHNGarageReconciler) ensureGarageBucketCR(ctx context.Context, claimNamespace, claimName, instanceNamespace string) error { + name := sliGarageBucketCRName(claimName) + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(garageBucketGVK) + + err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: claimNamespace}, obj) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + + obj = &unstructured.Unstructured{} + obj.SetGroupVersionKind(garageBucketGVK) + obj.SetName(name) + obj.SetNamespace(claimNamespace) + if err := unstructured.SetNestedField(obj.Object, claimName, "spec", "clusterRef", "name"); err != nil { + return fmt.Errorf("cannot set clusterRef name: %w", err) + } + if err := unstructured.SetNestedField(obj.Object, instanceNamespace, "spec", "clusterRef", "namespace"); err != nil { + return fmt.Errorf("cannot set clusterRef namespace: %w", err) + } + + return r.Create(ctx, obj) +} + +func (r *VSHNGarageReconciler) ensureGarageKeyCR(ctx context.Context, claimNamespace, claimName, instanceNamespace string) error { + name := sliGarageBucketCRName(claimName) + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(garageKeyGVK) + + err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: claimNamespace}, obj) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + + obj = &unstructured.Unstructured{} + obj.SetGroupVersionKind(garageKeyGVK) + obj.SetName(name) + obj.SetNamespace(claimNamespace) + if err := unstructured.SetNestedField(obj.Object, claimName, "spec", "clusterRef", "name"); err != nil { + return fmt.Errorf("cannot set clusterRef name: %w", err) + } + if err := unstructured.SetNestedField(obj.Object, instanceNamespace, "spec", "clusterRef", "namespace"); err != nil { + return fmt.Errorf("cannot set clusterRef namespace: %w", err) + } + bucketPerms := []interface{}{ + map[string]interface{}{ + "bucketRef": name, + "read": true, + "write": true, + }, + } + if err := unstructured.SetNestedSlice(obj.Object, bucketPerms, "spec", "bucketPermissions"); err != nil { + return fmt.Errorf("cannot set bucketPermissions: %w", err) + } + + return r.Create(ctx, obj) +} + +func (r *VSHNGarageReconciler) cleanupSLIBucketCR(ctx context.Context, inst *vshnv1.XVSHNGarage) error { + claimName := inst.Labels[slireconciler.ClaimNameLabel] + claimNamespace := inst.Labels[slireconciler.ClaimNamespaceLabel] + if claimName == "" || claimNamespace == "" { + return nil + } + + name := sliGarageBucketCRName(claimName) + + bucketObj := &unstructured.Unstructured{} + bucketObj.SetGroupVersionKind(garageBucketGVK) + bucketObj.SetName(name) + bucketObj.SetNamespace(claimNamespace) + if err := r.Delete(ctx, bucketObj); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete SLI GarageBucket CR: %w", err) + } + + keyObj := &unstructured.Unstructured{} + keyObj.SetGroupVersionKind(garageKeyGVK) + keyObj.SetName(name) + keyObj.SetNamespace(claimNamespace) + if err := r.Delete(ctx, keyObj); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete SLI GarageKey CR: %w", err) + } + + return nil +} + +func sliGarageBucketCRName(claimName string) string { + name := "vshn-sli-probe-" + claimName + if len(name) > 63 { + hash := fmt.Sprintf("%x", sha256.Sum256([]byte(name))) + name = name[:57] + "-" + hash[:5] + } + return name +} + +func (r *VSHNGarageReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&vshnv1.XVSHNGarage{}). + Complete(r) +} diff --git a/pkg/sliexporter/vshngarage_controller/vshngarage_controller_test.go b/pkg/sliexporter/vshngarage_controller/vshngarage_controller_test.go new file mode 100644 index 0000000000..4e00c78b6c --- /dev/null +++ b/pkg/sliexporter/vshngarage_controller/vshngarage_controller_test.go @@ -0,0 +1,317 @@ +package vshngaragecontroller + +import ( + "context" + "fmt" + "testing" + "time" + + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + cpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1" + "github.com/vshn/appcat/v4/pkg/sliexporter/probes" + slireconciler "github.com/vshn/appcat/v4/pkg/sliexporter/sli_reconciler" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + garageName = "test-garage" + garageClaimNS = "garagens" + garageInstanceNS = "vshn-garage-test-garage" + + _ probeManager = &fakeProbeManager{} +) + +type fakeProbeManager struct { + probers map[key]bool +} + +func newFakeProbeManager() *fakeProbeManager { + return &fakeProbeManager{ + probers: map[key]bool{}, + } +} + +type key string + +func (m *fakeProbeManager) StartProbe(p probes.Prober) { + m.probers[getFakeKey(p.GetInfo())] = true +} + +func (m *fakeProbeManager) StopProbe(p probes.ProbeInfo) { + m.probers[getFakeKey(p)] = false +} + +func getFakeKey(pi probes.ProbeInfo) key { + return key(fmt.Sprintf("%s; %s", pi.Service, pi.Name)) +} + +func TestGarageReconciler(t *testing.T) { + garage, ns := giveMeGarage(garageName, garageClaimNS) + + ct := metav1.Now().Add(-20 * time.Minute) + garage.CreationTimestamp = metav1.Time{Time: ct} + + r, manager, c := setupVSHNGarageTest(t, + garage, ns, + newTestBucketCreds(garageClaimNS, garageName), + newTestAdminToken(garageInstanceNS), + ) + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: garageName, + Namespace: garageClaimNS, + }, + } + + probeInfo := probes.ProbeInfo{ + Service: "VSHNGarage", + Name: garageName, + ClaimNamespace: garageClaimNS, + InstanceNamespace: garageInstanceNS, + } + + _, err := r.Reconcile(context.TODO(), req) + assert.NoError(t, err) + assert.True(t, manager.probers[getFakeKey(probeInfo)]) + + require.NoError(t, c.Delete(context.TODO(), garage)) + _, err = r.Reconcile(context.TODO(), req) + assert.NoError(t, err) + assert.False(t, manager.probers[getFakeKey(probeInfo)]) +} + +func TestVSHNGarage_Startup_NoCreds_Dont_Probe(t *testing.T) { + garage, ns := giveMeGarage(garageName, garageClaimNS) + + r, manager, _ := setupVSHNGarageTest(t, garage, ns) + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: garageName, + Namespace: garageClaimNS, + }, + } + + probeInfo := probes.ProbeInfo{ + Service: "VSHNGarage", + Name: garageName, + ClaimNamespace: garageClaimNS, + InstanceNamespace: garageInstanceNS, + } + + res, err := r.Reconcile(context.TODO(), req) + assert.NoError(t, err) + assert.Greater(t, res.RequeueAfter.Microseconds(), int64(0)) + assert.False(t, manager.probers[getFakeKey(probeInfo)]) +} + +func TestVSHNGarage_NoRef_Dont_Probe(t *testing.T) { + garage, ns := giveMeGarage(garageName, garageClaimNS) + garage.Spec.WriteConnectionSecretToReference.Name = "" + + r, manager, _ := setupVSHNGarageTest(t, garage, ns) + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: garageName, + Namespace: garageClaimNS, + }, + } + + pi := probes.ProbeInfo{ + Service: "VSHNGarage", + Name: garageName, + } + + _, err := r.Reconcile(context.TODO(), req) + assert.NoError(t, err) + assert.False(t, manager.probers[getFakeKey(pi)]) +} + +func TestVSHNGarage_Suspended_Dont_Probe(t *testing.T) { + garage, ns := giveMeGarage(garageName, garageClaimNS) + garage.Spec.Parameters.Instances = 0 + + ct := metav1.Now().Add(-20 * time.Minute) + garage.CreationTimestamp = metav1.Time{Time: ct} + + r, manager, _ := setupVSHNGarageTest(t, + garage, ns, + newTestBucketCreds(garageClaimNS, garageName), + newTestAdminToken(garageInstanceNS), + ) + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: garageName, + Namespace: garageClaimNS, + }, + } + + pi := probes.ProbeInfo{ + Service: "VSHNGarage", + Name: garageName, + } + + _, err := r.Reconcile(context.TODO(), req) + assert.NoError(t, err) + assert.False(t, manager.probers[getFakeKey(pi)]) +} + +func TestVSHNGarage_Suspended_Stop_Running_Probe(t *testing.T) { + garage, ns := giveMeGarage(garageName, garageClaimNS) + + ct := metav1.Now().Add(-20 * time.Minute) + garage.CreationTimestamp = metav1.Time{Time: ct} + + r, manager, c := setupVSHNGarageTest(t, + garage, ns, + newTestBucketCreds(garageClaimNS, garageName), + newTestAdminToken(garageInstanceNS), + ) + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: garageName, + Namespace: garageClaimNS, + }, + } + + pi := probes.ProbeInfo{ + Service: "VSHNGarage", + Name: garageName, + } + + _, err := r.Reconcile(context.TODO(), req) + assert.NoError(t, err) + assert.True(t, manager.probers[getFakeKey(pi)]) + + garage.Spec.Parameters.Instances = 0 + require.NoError(t, c.Update(context.TODO(), garage)) + _, err = r.Reconcile(context.TODO(), req) + assert.NoError(t, err) + assert.False(t, manager.probers[getFakeKey(pi)]) +} + +func giveMeGarage(name, claimNamespace string) (*vshnv1.XVSHNGarage, *corev1.Namespace) { + garage := &vshnv1.XVSHNGarage{ + TypeMeta: metav1.TypeMeta{ + Kind: "XVSHNGarage", + APIVersion: "vshn.appcat.vshn.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: claimNamespace, + CreationTimestamp: metav1.Now(), + Labels: map[string]string{ + slireconciler.ClaimNamespaceLabel: claimNamespace, + slireconciler.ClaimNameLabel: name, + }, + }, + Spec: vshnv1.XVSHNGarageSpec{ + Parameters: vshnv1.VSHNGarageParameters{ + Instances: 3, + }, + CompositionRef: cpv1.CompositionReference{Name: "vshngarage.vshn.appcat.vshn.io"}, + ResourceSpec: xpv1.ResourceSpec{ + WriteConnectionSecretToReference: &xpv1.SecretReference{ + Name: name, + Namespace: "vshn-garage-" + name, + }, + }, + }, + Status: vshnv1.XVSHNGarageStatus{ + VSHNGarageStatus: vshnv1.VSHNGarageStatus{ + InstanceNamespace: "vshn-garage-" + name, + }, + }, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: garage.GetInstanceNamespace(), + }, + } + + return garage, ns +} + +func setupVSHNGarageTest(t *testing.T, objs ...client.Object) (VSHNGarageReconciler, *fakeProbeManager, client.Client) { + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, vshnv1.AddToScheme(scheme)) + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + Build() + + manager := newFakeProbeManager() + r := VSHNGarageReconciler{ + Client: c, + Scheme: scheme, + ProbeManager: manager, + StartupGracePeriod: 5 * time.Minute, + GarageDialer: fakeGarageDialer, + ScClient: c, + } + + return r, manager, c +} + +func fakeGarageDialer(service, name, claimNamespace, instanceNamespace, organization, sla, compositionName, endpointURL, adminToken, bucketName string, ha bool, opts minio.Options) (*probes.VSHNGarage, error) { + return &probes.VSHNGarage{ + Service: service, + Name: name, + ClaimNamespace: claimNamespace, + InstanceNamespace: instanceNamespace, + Organization: organization, + HighAvailable: ha, + ServiceLevel: sla, + }, nil +} + +func newTestBucketCreds(namespace, claimName string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: sliGarageBucketCRName(claimName), + Namespace: namespace, + }, + Data: map[string][]byte{ + "endpoint": []byte("http://s3.garage.svc:3900"), + "access-key-id": []byte("GK1234567890abcdef"), + "secret-access-key": []byte("supersecret"), + }, + } +} + +func newTestAdminToken(namespace string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: adminTokenSecret, + Namespace: namespace, + }, + Data: map[string][]byte{ + "token": []byte("admin-bearer-token"), + }, + } +}