From f541227794ea23d7769754524ab4230350b01a71 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 03:34:43 -0300 Subject: [PATCH 01/14] Remove Wrong comment Signed-off-by: hamistao --- validation/longhorn/chartinstall/installation_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/validation/longhorn/chartinstall/installation_test.go b/validation/longhorn/chartinstall/installation_test.go index c62bc8734b..27f94c3e18 100644 --- a/validation/longhorn/chartinstall/installation_test.go +++ b/validation/longhorn/chartinstall/installation_test.go @@ -194,7 +194,6 @@ func (l *LonghornChartTestSuite) TestChartInstallStaticCustomConfig() { require.NoError(l.T(), err) require.NotEmpty(l.T(), nginxSpec.Template.Spec.Volumes) - // Even though the Longhorn default for number of replicas is 2, Rancher enforces its own default of 3. volumeName := nginxSpec.Template.Spec.Volumes[0].Name checkReplicasCommand := []string{"kubectl", "-n", charts.LonghornNamespace, "get", "volumes.longhorn.io", volumeName, `-o=jsonpath="{.spec.numberOfReplicas}"`} settingValue, err := kubectl.Command(l.standardUserClient, nil, l.cluster.ID, checkReplicasCommand, "") From 8f6d3587d63253904fce808b99839aeb9bc759b2 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 14:40:52 -0300 Subject: [PATCH 02/14] Minor refactorings Signed-off-by: hamistao --- actions/charts/longhorn.go | 10 +++--- actions/kubeapi/longhorn/longhorn.go | 48 ++++++++++------------------ validation/longhorn/longhorn_test.go | 19 ++++++----- 3 files changed, 31 insertions(+), 46 deletions(-) diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 89c8640cb3..0ffcf35f04 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -41,7 +41,7 @@ type LonghornGlobalSettingPut struct { // InstallLonghornChart installs the Longhorn chart on the cluster according to data on the payload. // Extra values can be passed in through the values argument. // This also waits for installation to complete and checks if the deployments are Ready. -func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values map[string]interface{}) error { +func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values map[string]any) error { // Callers pass a payload without Name; populate it so chart-action lifecycle and // failure logs carry the chart name. payload.Name = LonghornChartName @@ -108,10 +108,10 @@ func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID } uninstallPayload := &PayloadOpts{ - InstallOptions: InstallOptions{Cluster: &clusters.ClusterMeta{ID: clusterID}}, - Name: LonghornChartName, - Namespace: namespace, - Host: rancherHostname, + Cluster: &clusters.ClusterMeta{ID: clusterID}, + Name: LonghornChartName, + Namespace: namespace, + Host: rancherHostname, } bodyBytes, err := json.Marshal(NewChartUninstallAction()) diff --git a/actions/kubeapi/longhorn/longhorn.go b/actions/kubeapi/longhorn/longhorn.go index 39e7a4a007..9968a8a141 100644 --- a/actions/kubeapi/longhorn/longhorn.go +++ b/actions/kubeapi/longhorn/longhorn.go @@ -53,10 +53,8 @@ func GetNumberOfLonghornVolumes(client *rancher.Client, clusterID string) (int, func CreateS3LonghornBackupTarget(client *rancher.Client, clusterID string, awsCreds cloudcredentials.AmazonEC2CredentialConfig, region, bucketName string) (*longhorn.BackupTarget, error) { secretName := namegenerator.AppendRandomString("aws-creds") secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: charts.LonghornNamespace, - }, + Name: secretName, + Namespace: charts.LonghornNamespace, StringData: map[string]string{ "AWS_ACCESS_KEY_ID": awsCreds.AccessKey, "AWS_SECRET_ACCESS_KEY": awsCreds.SecretKey, @@ -164,14 +162,10 @@ func CreateLonghornVolumeBackup(client *rancher.Client, clusterID string, namesp } snapshot := longhorn.Snapshot{ - TypeMeta: metav1.TypeMeta{ - Kind: longhornTypes.LonghornKindSnapshot, - APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: namegenerator.AppendRandomString("backup-test"), - Namespace: namespace, - }, + Kind: longhornTypes.LonghornKindSnapshot, + APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version, + Name: namegenerator.AppendRandomString("backup-test"), + Namespace: namespace, Spec: longhorn.SnapshotSpec{ Volume: volumeName, CreateSnapshot: true, @@ -196,16 +190,12 @@ func CreateLonghornVolumeBackup(client *rancher.Client, clusterID string, namesp } backup := &longhorn.Backup{ - TypeMeta: metav1.TypeMeta{ - Kind: longhornTypes.LonghornKindBackup, - APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: namegenerator.AppendRandomString("backup-test"), - Namespace: namespace, - Labels: map[string]string{ - "backup-volume": volumeName, - }, + Kind: longhornTypes.LonghornKindBackup, + APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version, + Name: namegenerator.AppendRandomString("backup-test"), + Namespace: namespace, + Labels: map[string]string{ + "backup-volume": volumeName, }, Spec: longhorn.BackupSpec{ SnapshotName: snapshot.Name, @@ -273,7 +263,7 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b } backupVolumeSelector := labels.SelectorFromSet(longhornTypes.GetBackupVolumeWithBackupTargetLabels(backup.Status.BackupTargetName, backup.Status.VolumeName)) - err = wait.PollUntilContextTimeout(context.Background(), defaults.FiveSecondTimeout, defaults.FiveMinuteTimeout, true, func(context.Context) (bool, error) { + err = wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, defaults.FifteenMinuteTimeout, true, func(context.Context) (bool, error) { var backupVolumes longhorn.BackupVolumeList err := backupVolumeController.Client().List(context.Background(), backup.Namespace, &backupVolumes, metav1.ListOptions{LabelSelector: backupVolumeSelector.String()}) if err != nil { @@ -288,14 +278,10 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b // This volume is based on the docs: https://longhorn.io/docs/1.11.1/snapshots-and-backups/backup-and-restore/restore-from-a-backup/ volume := &longhorn.Volume{ - TypeMeta: metav1.TypeMeta{ - Kind: longhornTypes.LonghornKindVolume, - APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: namegenerator.AppendRandomString("restored-" + backup.Status.VolumeName), - Namespace: backup.Namespace, - }, + Kind: longhornTypes.LonghornKindVolume, + APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version, + Name: namegenerator.AppendRandomString("restored-" + backup.Status.VolumeName), + Namespace: backup.Namespace, Spec: longhorn.VolumeSpec{ Size: int64(volumeSize), FromBackup: backup.Status.URL, diff --git a/validation/longhorn/longhorn_test.go b/validation/longhorn/longhorn_test.go index b3bc054883..28d596bf60 100644 --- a/validation/longhorn/longhorn_test.go +++ b/validation/longhorn/longhorn_test.go @@ -62,12 +62,13 @@ var ( type LonghornTestSuite struct { suite.Suite - client *rancher.Client - standardUserClient *rancher.Client - session *session.Session - longhornTestConfig longhorn.TestConfig - cluster *clusters.ClusterMeta - project *management.Project + client *rancher.Client + standardUserClient *rancher.Client + session *session.Session + longhornTestConfig longhorn.TestConfig + cluster *clusters.ClusterMeta + project *management.Project + longhornPreInstalled bool } func (l *LonghornTestSuite) TearDownSuite() { @@ -272,10 +273,8 @@ func (l *LonghornTestSuite) TestVolumeEncryption() { secretName := namegenerator.AppendRandomString(LonghornEncryptionSecretBaseName) secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: charts.LonghornNamespace, - }, + Name: secretName, + Namespace: charts.LonghornNamespace, StringData: map[string]string{ "CRYPTO_KEY_VALUE": string(key), "CRYPTO_KEY_PROVIDER": "secret", From 2aaace11e94ad2710e0e231f2b0ed809eb5d5d17 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 03:34:56 -0300 Subject: [PATCH 03/14] Wait for workload readiness Signed-off-by: hamistao --- actions/charts/ranchermonitoring.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/actions/charts/ranchermonitoring.go b/actions/charts/ranchermonitoring.go index 40a64e3d8f..88ca736c2b 100644 --- a/actions/charts/ranchermonitoring.go +++ b/actions/charts/ranchermonitoring.go @@ -9,6 +9,7 @@ import ( catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" "github.com/rancher/shepherd/extensions/clusters" "github.com/rancher/shepherd/extensions/defaults" "github.com/rancher/shepherd/extensions/kubectl" @@ -160,7 +161,21 @@ func InstallRancherMonitoringChart(client *rancher.Client, installOptions *Insta if err != nil { return err } - return nil + + // Wait for the monitoring workloads to be Ready so the chart is actually serving + // (Prometheus and Alertmanager run as StatefulSets; the operator and Grafana as Deployments). + // Without this, callers can race against Prometheus not yet scraping any targets. + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, RancherMonitoringNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, RancherMonitoringNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitStatefulSets(client, installOptions.Cluster.ID, RancherMonitoringNamespace, metav1.ListOptions{}) } // newMonitoringChartInstallAction is a private helper function that returns chart install action with monitoring and payload options. From 224d0643d0ef8f365ccfc3154dfe786a41209db7 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 14:41:51 -0300 Subject: [PATCH 04/14] Create SericeMonitor in monitoring namespace Signed-off-by: hamistao --- validation/longhorn/longhorn_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/longhorn/longhorn_test.go b/validation/longhorn/longhorn_test.go index 28d596bf60..346bad8f7a 100644 --- a/validation/longhorn/longhorn_test.go +++ b/validation/longhorn/longhorn_test.go @@ -483,7 +483,7 @@ func (l *LonghornTestSuite) TestMonitoringIntegration() { } serviceMonitorName := namegenerator.AppendRandomString("longhorn-monitor") - _, err = monitoring.CreateServiceMonitor(l.standardUserClient, l.cluster.ID, serviceMonitorName, charts.LonghornNamespace, serviceMonitorSpec) + _, err = monitoring.CreateServiceMonitor(l.standardUserClient, l.cluster.ID, serviceMonitorName, charts.RancherMonitoringNamespace, serviceMonitorSpec) require.NoError(l.T(), err) l.T().Logf("ServiceMonitor %s for Longhorn created", serviceMonitorName) From 48ebdfcef2621528e57ba2e27ba8dab46852bcf2 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 14:42:06 -0300 Subject: [PATCH 05/14] Logging improvement Signed-off-by: hamistao --- validation/longhorn/longhorn_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/longhorn/longhorn_test.go b/validation/longhorn/longhorn_test.go index 346bad8f7a..a678a997e8 100644 --- a/validation/longhorn/longhorn_test.go +++ b/validation/longhorn/longhorn_test.go @@ -459,7 +459,7 @@ func (l *LonghornTestSuite) TestMonitoringIntegration() { ProjectID: string(monitoringProject.ID), } - l.T().Log("Installing monitoring chart") + l.T().Logf("Installing monitoring chart version %s", latestMonitoringVersion) err = charts.InstallRancherMonitoringChart(l.standardUserClient, chartInstallOptions, &charts.RancherMonitoringOpts{ IngressNginx: true, ControllerManager: true, From 5b96691497ccd2866f9fa6474f0b44d237a398ad Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 03:35:53 -0300 Subject: [PATCH 06/14] Use `RancherMonitoringNamespace` variable Signed-off-by: hamistao --- actions/charts/action_diagnostics_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/actions/charts/action_diagnostics_test.go b/actions/charts/action_diagnostics_test.go index e9698280b9..b0d461a011 100644 --- a/actions/charts/action_diagnostics_test.go +++ b/actions/charts/action_diagnostics_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/rancher/shepherd/clients/rancher/catalog" - "github.com/rancher/shepherd/clients/rancher/v1" + v1 "github.com/rancher/shepherd/clients/rancher/v1" "github.com/rancher/shepherd/extensions/clusters" scheme "github.com/rancher/shepherd/pkg/generated/clientset/versioned/scheme" "github.com/sirupsen/logrus" @@ -53,7 +53,7 @@ func chartActionTestOpts() *PayloadOpts { return &PayloadOpts{ InstallOptions: InstallOptions{Cluster: &clusters.ClusterMeta{ID: "c-test-local"}}, Name: "rancher-monitoring", - Namespace: "cattle-monitoring-system", + Namespace: RancherMonitoringNamespace, } } @@ -386,7 +386,7 @@ func TestChartActionRequestBuildersDisableClientGoRetries(t *testing.T) { func TestLogChartActionFailureNilClientIsNoop(t *testing.T) { // Must not panic or log diagnostics when there is no client to read with. - LogChartActionFailure(context.Background(), nil, "rancher-charts", "c-local", "cattle-monitoring-system", []string{"rancher-monitoring"}) + LogChartActionFailure(context.Background(), nil, "rancher-charts", "c-local", RancherMonitoringNamespace, []string{"rancher-monitoring"}) } func TestChartOperationStatusForLogRedactsSensitiveFields(t *testing.T) { @@ -396,7 +396,7 @@ func TestChartOperationStatusForLogRedactsSensitiveFields(t *testing.T) { "chart": "rancher-monitoring", "version": "102.0.0", "releaseName": "rancher-monitoring", - "namespace": "cattle-monitoring-system", + "namespace": RancherMonitoringNamespace, "podName": "helm-operation-abc123", "podCreated": true, "token": "kubeconfig-u-abcdef-secrettoken", From 23b9af89b0c459d53328bc5f4dcdaaa51dc5bc01 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 14:41:21 -0300 Subject: [PATCH 07/14] Avoid issues with network policies Signed-off-by: hamistao --- actions/charts/longhorn.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 0ffcf35f04..2506c63abd 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -23,10 +23,13 @@ var ( LonghornStaticStorageClass = "longhorn-static" LonghornNamespace = "longhorn-system" LonghornChartName = "longhorn" - enableDeletionSetting = map[string]any{ + longhornChartSettings = map[string]any{ "defaultSettings": map[string]any{ "deletingConfirmationFlag": true, }, + "networkPolicies": map[string]any{ + "restrictInternalTraffic": false, + }, } ) @@ -53,17 +56,22 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma // If no specific value for setting deletingConfirmationFlag was provided, default to have it enabled so cleanup works as expected. if values == nil { - values = enableDeletionSetting + values = longhornChartSettings } else { defaultSettings, ok := values["defaultSettings"].(map[string]any) if !ok { return errors.New(`Provided values map has invalid value for "defaultSettings"`) } - _, ok = defaultSettings["deletingConfirmationFlag"] + defaultSettings["deletingConfirmationFlag"] = true + + networkPolicies, ok := values["networkPolicies"].(map[string]any) if !ok { - defaultSettings["deletingConfirmationFlag"] = true + networkPolicies = map[string]any{} + values["networkPolicies"] = networkPolicies } + + networkPolicies["restrictInternalTraffic"] = false } chartInstalls := []types.ChartInstall{ From c3d29de7c571e1663efb3792875fb1fafd135c43 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 14:42:24 -0300 Subject: [PATCH 08/14] Warn on the presence of longhorn network policies Signed-off-by: hamistao --- validation/longhorn/longhorn_test.go | 41 ++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/validation/longhorn/longhorn_test.go b/validation/longhorn/longhorn_test.go index a678a997e8..997a279e42 100644 --- a/validation/longhorn/longhorn_test.go +++ b/validation/longhorn/longhorn_test.go @@ -58,6 +58,8 @@ var ( LonghornEncryptionStorageClassBaseName = "longhorn-crypto-global" volumeBytesPrometheusQueryTemplate = "longhorn_volume_capacity_bytes{volume=\"%s\"}" numberVolumesPrometheusQuery = "count(longhorn_volume_capacity_bytes) OR on() vector(0)" // Return an empty list instead of nil with OR on() vector(0). + longhornNetworkPolicySteveType = "networking.k8s.io.networkpolicy" + longhornManagerNetworkPolicyName = "longhorn-manager" ) type LonghornTestSuite struct { @@ -106,6 +108,7 @@ func (l *LonghornTestSuite) SetupSuite() { chart, err := shepherdCharts.GetChartStatus(l.standardUserClient, l.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) require.NoError(l.T(), err) + l.longhornPreInstalled = chart.IsAlreadyInstalled if !chart.IsAlreadyInstalled { // Get latest versions of longhorn latestLonghornVersion, err := l.standardUserClient.Catalog.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) @@ -114,11 +117,9 @@ func (l *LonghornTestSuite) SetupSuite() { payloadOpts := charts.PayloadOpts{ Namespace: charts.LonghornNamespace, Host: l.client.RancherConfig.Host, - InstallOptions: charts.InstallOptions{ - Cluster: l.cluster, - Version: latestLonghornVersion, - ProjectID: l.project.ID, - }, + Cluster: l.cluster, + Version: latestLonghornVersion, + ProjectID: l.project.ID, } l.T().Logf("Installing Lonhgorn chart in cluster [%v] with latest version [%v] in project [%v] and namespace [%v]", l.cluster.Name, payloadOpts.Version, l.project.Name, payloadOpts.Namespace) @@ -470,6 +471,10 @@ func (l *LonghornTestSuite) TestMonitoringIntegration() { require.NoError(l.T(), err) } + if l.longhornPreInstalled { + l.warnIfLonghornManagerNetworkPolicyExists() + } + serviceMonitorSpec := monitoringv1.ServiceMonitorSpec{ Selector: metav1.LabelSelector{ MatchLabels: map[string]string{"app": "longhorn-manager"}, @@ -516,6 +521,32 @@ func (l *LonghornTestSuite) TestMonitoringIntegration() { require.Equal(l.T(), previousNumberOfVolumes+1, value) } +// warnIfLonghornManagerNetworkPolicyExists logs a warning when a pre-installed Longhorn has the +// longhorn-manager NetworkPolicy (created when networkPolicies.restrictInternalTraffic is true), +// since it blocks Prometheus from scraping the Longhorn metrics endpoint. +func (l *LonghornTestSuite) warnIfLonghornManagerNetworkPolicyExists() { + steveClient, err := l.client.Steve.ProxyDownstream(l.cluster.ID) + if err != nil { + l.T().Logf("WARNING: could not check for Longhorn NetworkPolicies: %v", err) + return + } + + policies, err := steveClient.SteveType(longhornNetworkPolicySteveType).NamespacedSteveClient(charts.LonghornNamespace).List(nil) + if err != nil { + l.T().Logf("WARNING: could not check for Longhorn NetworkPolicies: %v", err) + return + } + + for _, policy := range policies.Data { + if policy.Name == longhornManagerNetworkPolicyName { + l.T().Logf("WARNING: the pre-installed Longhorn has the NetworkPolicy %s/%s, which blocks Prometheus from scraping Longhorn metrics. "+ + "This test will likely fail. Reinstall Longhorn with networkPolicies.restrictInternalTraffic=false or allow ingress from %s to port 9500.", + charts.LonghornNamespace, longhornManagerNetworkPolicyName, charts.RancherMonitoringNamespace) + return + } + } +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestLonghornTestSuite(t *testing.T) { From 1ec6c4c9dc7602c2959a1f62a07548afd798beb5 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 18:04:03 -0300 Subject: [PATCH 09/14] Prevent `TestVolumeEncryption` failures Signed-off-by: hamistao --- actions/kubeapi/longhorn/longhorn.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/actions/kubeapi/longhorn/longhorn.go b/actions/kubeapi/longhorn/longhorn.go index 9968a8a141..ea0bb163a5 100644 --- a/actions/kubeapi/longhorn/longhorn.go +++ b/actions/kubeapi/longhorn/longhorn.go @@ -276,6 +276,10 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b return fmt.Errorf("backup volume for volume %s on backup target %s was not created: %w", backup.Status.VolumeName, backup.Status.BackupTargetName, err) } + // Backups of encrypted volumes must be restored into encrypted volumes. Longhorn reports the backup size without + // the LUKS header and adds it back to the backend size of encrypted volumes, so the size can be used as is. + encrypted := backup.Status.Labels[longhornTypes.LonghornLabelVolumeEncrypted] == longhornTypes.LonghornLabelValueEnabled + // This volume is based on the docs: https://longhorn.io/docs/1.11.1/snapshots-and-backups/backup-and-restore/restore-from-a-backup/ volume := &longhorn.Volume{ Kind: longhornTypes.LonghornKindVolume, @@ -283,6 +287,7 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b Name: namegenerator.AppendRandomString("restored-" + backup.Status.VolumeName), Namespace: backup.Namespace, Spec: longhorn.VolumeSpec{ + Encrypted: encrypted, Size: int64(volumeSize), FromBackup: backup.Status.URL, Frontend: "blockdev", From 8a96583edc8b553acc89206ffbde5c13160d4c5b Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 18:04:41 -0300 Subject: [PATCH 10/14] Fix Backup cleanup Signed-off-by: hamistao --- actions/kubeapi/longhorn/longhorn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/kubeapi/longhorn/longhorn.go b/actions/kubeapi/longhorn/longhorn.go index ea0bb163a5..5bc7b0777f 100644 --- a/actions/kubeapi/longhorn/longhorn.go +++ b/actions/kubeapi/longhorn/longhorn.go @@ -209,7 +209,7 @@ func CreateLonghornVolumeBackup(client *rancher.Client, clusterID string, namesp } client.Session.RegisterCleanupFunc(func() error { - return backupController.Client().Delete(context.Background(), namespace, snapshot.Name, metav1.DeleteOptions{}) + return backupController.Client().Delete(context.Background(), namespace, backup.Name, metav1.DeleteOptions{}) }) err = wait.PollUntilContextTimeout(context.Background(), defaults.FiveHundredMillisecondTimeout, defaults.FiveMinuteTimeout, false, func(context.Context) (bool, error) { From 990d5ff8ca2bc80a9102659679cca7fa25978831 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 22 Oct 2026 18:04:56 -0300 Subject: [PATCH 11/14] Improve error handling Signed-off-by: hamistao --- actions/kubeapi/longhorn/longhorn.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/actions/kubeapi/longhorn/longhorn.go b/actions/kubeapi/longhorn/longhorn.go index 5bc7b0777f..0e0ad2d556 100644 --- a/actions/kubeapi/longhorn/longhorn.go +++ b/actions/kubeapi/longhorn/longhorn.go @@ -310,10 +310,19 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b return false, err } + for _, condition := range volume.Status.Conditions { + if condition.Type == longhorn.VolumeConditionTypeRestore && condition.Reason == longhorn.VolumeConditionReasonRestoreFailure { + return false, fmt.Errorf("restore of volume %s from backup %s failed: %s", volume.Name, backup.Name, condition.Message) + } + } + return !volume.Status.RestoreRequired && volume.Status.State == longhorn.VolumeStateDetached, nil }) + if err != nil { + return fmt.Errorf("volume %s restored from backup %s did not become ready (state: %s, restoreRequired: %t): %w", volume.Name, backup.Name, volume.Status.State, volume.Status.RestoreRequired, err) + } - return err + return nil } // DeleteLonghornVolume deletes the specified volume.longhorn.io resource. From 328b469d6e6a118b82986b6e9aa8120a0ee15377 Mon Sep 17 00:00:00 2001 From: hamistao Date: Mon, 26 Oct 2026 12:09:06 -0300 Subject: [PATCH 12/14] Use `LonghornCRDChartName` Signed-off-by: hamistao --- actions/charts/longhorn.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 2506c63abd..b2da5e3f69 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -23,6 +23,7 @@ var ( LonghornStaticStorageClass = "longhorn-static" LonghornNamespace = "longhorn-system" LonghornChartName = "longhorn" + LonghornCRDChartName = "longhorn-crd" longhornChartSettings = map[string]any{ "defaultSettings": map[string]any{ "deletingConfirmationFlag": true, @@ -75,7 +76,7 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma } chartInstalls := []types.ChartInstall{ - *NewChartInstall(LonghornChartName+"-crd", payload.Version, payload.Cluster.ID, payload.Cluster.Name, payload.Host, catalog.RancherChartRepo, payload.ProjectID, payload.DefaultRegistry, nil), + *NewChartInstall(LonghornCRDChartName, payload.Version, payload.Cluster.ID, payload.Cluster.Name, payload.Host, catalog.RancherChartRepo, payload.ProjectID, payload.DefaultRegistry, nil), *NewChartInstall(LonghornChartName, payload.Version, payload.Cluster.ID, payload.Cluster.Name, payload.Host, catalog.RancherChartRepo, payload.ProjectID, payload.DefaultRegistry, values), } @@ -85,7 +86,7 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma if err != nil { return err } - err = ChartActionWithRetry(context.TODO(), client, verbInstall, &payload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbInstall, bodyBytes)) + err = ChartActionWithRetry(context.TODO(), client, verbInstall, &payload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornCRDChartName}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbInstall, bodyBytes)) if err != nil { return err } @@ -141,12 +142,12 @@ func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID if err != nil { return err } - err = ChartActionWithRetry(context.TODO(), client, verbUninstall, uninstallPayload, "", []string{LonghornChartName + "-crd"}, buildAppUninstallRequest(catalogClient, namespace, LonghornChartName+"-crd", crdBodyBytes)) + err = ChartActionWithRetry(context.TODO(), client, verbUninstall, uninstallPayload, "", []string{LonghornCRDChartName}, buildAppUninstallRequest(catalogClient, namespace, LonghornCRDChartName, crdBodyBytes)) if err != nil { return err } - err = waitUninstallation(catalogClient, namespace, LonghornChartName+"-crd") + err = waitUninstallation(catalogClient, namespace, LonghornCRDChartName) if err != nil { return err } From 6ef181de6d0af810f35ace7acb284ba8fa39a931 Mon Sep 17 00:00:00 2001 From: hamistao Date: Tue, 27 Oct 2026 02:21:21 -0300 Subject: [PATCH 13/14] Ensure cluster repo is updated before installing charts Signed-off-by: hamistao --- actions/charts/repo.go | 48 ++++++++++++++++++++++++ validation/charts/alerting_test.go | 49 +++++++++++++++---------- validation/charts/installations_test.go | 2 + validation/charts/logging_test.go | 12 +++--- validation/charts/monitoring_test.go | 19 ++++------ validation/charts/neuvector_test.go | 12 +++--- validation/longhorn/longhorn_test.go | 6 +++ 7 files changed, 104 insertions(+), 44 deletions(-) create mode 100644 actions/charts/repo.go diff --git a/actions/charts/repo.go b/actions/charts/repo.go new file mode 100644 index 0000000000..d499492142 --- /dev/null +++ b/actions/charts/repo.go @@ -0,0 +1,48 @@ +package charts + +import ( + "context" + "time" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kwait "k8s.io/apimachinery/pkg/util/wait" +) + +// UpdateClusterRepo refreshes the indicated cluster, ensuring that the newest versions are available. +// Doesn't return and error and logs a warning if updating the cluster fails. +func UpdateClusterRepo(adminClient *rancher.Client, repoName string) { + err := updateClusterRepo(adminClient.Catalog, repoName) + + if err != nil { + logrus.Warnf("Failed to refresh cluster repo %s: %s", repoName, err.Error()) + } else { + logrus.Infof("Successfully refreshed cluster repo %s", repoName) + } +} + +func updateClusterRepo(catalogClient *catalog.Client, repoName string) error { + clusterRepo, err := catalogClient.ClusterRepos().Get(context.Background(), repoName, metav1.GetOptions{}) + if err != nil { + return err + } + + forceUpdate := metav1.Now() + clusterRepo.Spec.ForceUpdate = &forceUpdate + + _, err = catalogClient.ClusterRepos().Update(context.Background(), clusterRepo, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return kwait.PollUntilContextTimeout(context.Background(), 500*time.Millisecond, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + repo, err := catalogClient.ClusterRepos().Get(ctx, repoName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return !repo.Status.DownloadTime.Before(&forceUpdate), nil + }) +} diff --git a/validation/charts/alerting_test.go b/validation/charts/alerting_test.go index 8320d0d3e0..0d6330c32b 100644 --- a/validation/charts/alerting_test.go +++ b/validation/charts/alerting_test.go @@ -52,31 +52,40 @@ func (a *AlertingTestSuite) SetupSuite() { } func (a *AlertingTestSuite) TestAlertingInstallation() { - latestAlertingVersion, err := a.client.Catalog.GetLatestChartVersion( - charts.RancherAlertingName, - catalog.RancherChartRepo, - ) + alertingChart, err := shepherdCharts.GetChartStatus(a.client, a.cluster.ID, charts.RancherAlertingNamespace, charts.RancherAlertingName) require.NoError(a.T(), err) - installOptions := &charts.InstallOptions{ - Cluster: a.cluster, - Version: latestAlertingVersion, - ProjectID: a.project.ID, - } + if !alertingChart.IsAlreadyInstalled { + a.T().Skip("Alerting chart is already installed") - featureOptions := &charts.RancherAlertingOpts{ - SMS: true, - Teams: false, - } + latestAlertingVersion, err := a.client.Catalog.GetLatestChartVersion( + charts.RancherAlertingName, + catalog.RancherChartRepo, + ) + require.NoError(a.T(), err) - a.T().Logf( - "Installing Rancher Alerting chart on cluster [%s] with version [%s]", - a.cluster.Name, - latestAlertingVersion, - ) + charts.UpdateClusterRepo(a.client, catalog.RancherChartRepo) - err = charts.InstallRancherAlertingChart(a.client, installOptions, featureOptions) - require.NoError(a.T(), err) + installOptions := &charts.InstallOptions{ + Cluster: a.cluster, + Version: latestAlertingVersion, + ProjectID: a.project.ID, + } + + featureOptions := &charts.RancherAlertingOpts{ + SMS: true, + Teams: false, + } + + a.T().Logf( + "Installing Rancher Alerting chart on cluster [%s] with version [%s]", + a.cluster.Name, + latestAlertingVersion, + ) + + err = charts.InstallRancherAlertingChart(a.client, installOptions, featureOptions) + require.NoError(a.T(), err) + } a.T().Logf( "Waiting for alerting deployments to become ready in namespace [%s]", diff --git a/validation/charts/installations_test.go b/validation/charts/installations_test.go index 7a9d28a5dc..548c37b038 100644 --- a/validation/charts/installations_test.go +++ b/validation/charts/installations_test.go @@ -64,6 +64,8 @@ func (i *InstallationTestSuite) SetupSuite() { require.NoError(i.T(), err) require.Equal(i.T(), createdProject.Name, exampleAppProjectName) i.project = createdProject + + charts.UpdateClusterRepo(i.client, catalog.RancherChartRepo) } func (i *InstallationTestSuite) TestInstallMonitoringChart() { diff --git a/validation/charts/logging_test.go b/validation/charts/logging_test.go index 8dc3393db2..a6da0eb01a 100644 --- a/validation/charts/logging_test.go +++ b/validation/charts/logging_test.go @@ -58,11 +58,13 @@ func (l *LoggingTestSuite) SetupSuite() { } func (l *LoggingTestSuite) TestLoggingInstallation() { - l.T().Logf("Resolving latest chart version for [%s] from repository [%s]", charts.LonghornNamespace, charts.LonghornChartName) longhornChart, err := shepherdCharts.GetChartStatus(l.client, l.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) require.NoError(l.T(), err) if !longhornChart.IsAlreadyInstalled { + charts.UpdateClusterRepo(l.client, catalog.RancherChartRepo) + + l.T().Logf("Resolving latest chart version for [%s] from repository [%s]", charts.LonghornChartName, catalog.RancherChartRepo) // Get latest versions of longhorn latestLonghornVersion, err := l.client.Catalog.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(l.T(), err) @@ -70,11 +72,9 @@ func (l *LoggingTestSuite) TestLoggingInstallation() { payloadOpts := charts.PayloadOpts{ Namespace: charts.LonghornNamespace, Host: l.client.RancherConfig.Host, - InstallOptions: charts.InstallOptions{ - Cluster: l.cluster, - Version: latestLonghornVersion, - ProjectID: l.project.ID, - }, + Cluster: l.cluster, + Version: latestLonghornVersion, + ProjectID: l.project.ID, } l.T().Logf("Installing Longhorn chart in cluster [%v] with latest version [%v] in project [%v] and namespace [%v]", l.cluster.Name, payloadOpts.Version, l.project.Name, payloadOpts.Namespace) diff --git a/validation/charts/monitoring_test.go b/validation/charts/monitoring_test.go index 6b04c07f90..f06f6ffb55 100644 --- a/validation/charts/monitoring_test.go +++ b/validation/charts/monitoring_test.go @@ -133,15 +133,10 @@ func (m *MonitoringTestSuite) TestMonitoringChart() { require.NoError(m.T(), err) if !initialMonitoringChart.IsAlreadyInstalled { + charts.UpdateClusterRepo(m.client, catalog.RancherChartRepo) + m.T().Log("Installing monitoring chart") - err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { - // Check if chart was already installed by a previous retry attempt - status, statusErr := extencharts.GetChartStatus(client, m.project.ClusterID, charts.RancherMonitoringNamespace, charts.RancherMonitoringName) - if statusErr == nil && status.IsAlreadyInstalled { - return nil - } - return charts.InstallRancherMonitoringChart(client, m.chartInstallOptions, m.chartFeatureOptions) - }) + err = charts.InstallRancherMonitoringChart(client, m.chartInstallOptions, m.chartFeatureOptions) require.NoError(m.T(), err) m.T().Log("Waiting monitoring chart deployments to have expected number of available replicas") @@ -213,10 +208,8 @@ func (m *MonitoringTestSuite) TestMonitoringChart() { m.T().Log("Creating node port service for webhook receiver deployment") webhookServiceTemplate := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: webhookReceiverServiceName, - Namespace: webhookReceiverNamespace.Name, - }, + Name: webhookReceiverServiceName, + Namespace: webhookReceiverNamespace.Name, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeNodePort, Ports: []corev1.ServicePort{ @@ -345,6 +338,8 @@ func (m *MonitoringTestSuite) TestUpgradeMonitoringChart() { } if !initialMonitoringChart.IsAlreadyInstalled { + charts.UpdateClusterRepo(m.client, catalog.RancherChartRepo) + m.T().Log("Installing monitoring chart with the last but one version") err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { status, statusErr := extencharts.GetChartStatus(client, m.project.ClusterID, charts.RancherMonitoringNamespace, charts.RancherMonitoringName) diff --git a/validation/charts/neuvector_test.go b/validation/charts/neuvector_test.go index b44cc04af1..df6744bbec 100644 --- a/validation/charts/neuvector_test.go +++ b/validation/charts/neuvector_test.go @@ -62,12 +62,10 @@ func (n *NeuVectorTestSuite) SetupSuite() { require.NoError(n.T(), err) n.chartInstallOptions = &actionsCharts.PayloadOpts{ - Namespace: actionsCharts.NeuVectorNamespace, - InstallOptions: actionsCharts.InstallOptions{ - Cluster: n.cluster, - Version: latestVersion, - ProjectID: n.project.ID, - }, + Namespace: actionsCharts.NeuVectorNamespace, + Cluster: n.cluster, + Version: latestVersion, + ProjectID: n.project.ID, DefaultRegistry: registrySetting.Value, } } @@ -84,6 +82,8 @@ func (n *NeuVectorTestSuite) TestNeuVectorChartInstallation() { require.NoError(n.T(), err) if !neuVectorChartStatus.IsAlreadyInstalled { + actionsCharts.UpdateClusterRepo(n.client, catalog.RancherChartRepo) + n.T().Logf("Installing NeuVector on cluster [%s] with version [%s]", n.cluster.Name, n.chartInstallOptions.Version) err = actionsCharts.InstallNeuVectorChart(client, *n.chartInstallOptions) require.NoError(n.T(), err) diff --git a/validation/longhorn/longhorn_test.go b/validation/longhorn/longhorn_test.go index 997a279e42..a1c9fddbf7 100644 --- a/validation/longhorn/longhorn_test.go +++ b/validation/longhorn/longhorn_test.go @@ -110,6 +110,8 @@ func (l *LonghornTestSuite) SetupSuite() { l.longhornPreInstalled = chart.IsAlreadyInstalled if !chart.IsAlreadyInstalled { + charts.UpdateClusterRepo(l.client, catalog.RancherChartRepo) + // Get latest versions of longhorn latestLonghornVersion, err := l.standardUserClient.Catalog.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(l.T(), err) @@ -446,6 +448,10 @@ func (l *LonghornTestSuite) TestMonitoringIntegration() { require.NoError(l.T(), err) if !initialMonitoringChart.IsAlreadyInstalled { + if l.longhornPreInstalled { + charts.UpdateClusterRepo(l.client, catalog.RancherChartRepo) + } + // Get latest versions of the monitoring chart latestMonitoringVersion, err := l.standardUserClient.Catalog.GetLatestChartVersion(charts.RancherMonitoringName, catalog.RancherChartRepo) require.NoError(l.T(), err) From 2c62e23bb33dacc45571afc69e887f4e44567344 Mon Sep 17 00:00:00 2001 From: hamistao Date: Fri, 30 Oct 2026 23:27:51 -0300 Subject: [PATCH 14/14] Address review comments Signed-off-by: hamistao --- actions/charts/action_diagnostics_test.go | 6 +++--- actions/charts/longhorn.go | 8 +++++--- actions/charts/ranchermonitoring.go | 7 +++---- validation/charts/alerting_test.go | 2 -- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/actions/charts/action_diagnostics_test.go b/actions/charts/action_diagnostics_test.go index b0d461a011..67dc4b355d 100644 --- a/actions/charts/action_diagnostics_test.go +++ b/actions/charts/action_diagnostics_test.go @@ -51,9 +51,9 @@ func newChartActionTestDoer(t *testing.T, handler http.Handler) (chartActionDoer func chartActionTestOpts() *PayloadOpts { return &PayloadOpts{ - InstallOptions: InstallOptions{Cluster: &clusters.ClusterMeta{ID: "c-test-local"}}, - Name: "rancher-monitoring", - Namespace: RancherMonitoringNamespace, + Cluster: &clusters.ClusterMeta{ID: "c-test-local"}, + Name: "rancher-monitoring", + Namespace: RancherMonitoringNamespace, } } diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index b2da5e3f69..7409203856 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -64,12 +64,14 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma return errors.New(`Provided values map has invalid value for "defaultSettings"`) } - defaultSettings["deletingConfirmationFlag"] = true + _, ok = defaultSettings["deletingConfirmationFlag"] + if !ok { + defaultSettings["deletingConfirmationFlag"] = true + } networkPolicies, ok := values["networkPolicies"].(map[string]any) if !ok { - networkPolicies = map[string]any{} - values["networkPolicies"] = networkPolicies + return errors.New(`Provided values map has invalid value for "networkPolicies"`) } networkPolicies["restrictInternalTraffic"] = false diff --git a/actions/charts/ranchermonitoring.go b/actions/charts/ranchermonitoring.go index 88ca736c2b..be65e56cb4 100644 --- a/actions/charts/ranchermonitoring.go +++ b/actions/charts/ranchermonitoring.go @@ -162,10 +162,9 @@ func InstallRancherMonitoringChart(client *rancher.Client, installOptions *Insta return err } - // Wait for the monitoring workloads to be Ready so the chart is actually serving - // (Prometheus and Alertmanager run as StatefulSets; the operator and Grafana as Deployments). - // Without this, callers can race against Prometheus not yet scraping any targets. - err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, RancherMonitoringNamespace, metav1.ListOptions{}) + // Wait for the monitoring workloads to be Ready so the chart is actually serving. + releaseSelector := metav1.ListOptions{LabelSelector: "app.kubernetes.io/instance=" + RancherMonitoringName} + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, RancherMonitoringNamespace, releaseSelector) if err != nil { return err } diff --git a/validation/charts/alerting_test.go b/validation/charts/alerting_test.go index 0d6330c32b..c1ceddf980 100644 --- a/validation/charts/alerting_test.go +++ b/validation/charts/alerting_test.go @@ -56,8 +56,6 @@ func (a *AlertingTestSuite) TestAlertingInstallation() { require.NoError(a.T(), err) if !alertingChart.IsAlreadyInstalled { - a.T().Skip("Alerting chart is already installed") - latestAlertingVersion, err := a.client.Catalog.GetLatestChartVersion( charts.RancherAlertingName, catalog.RancherChartRepo,