Fix Longhorn tests

actions/charts/action_diagnostics_test.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/rancher/shepherd/clients/rancher/catalog"
-	"github.com/rancher/shepherd/clients/rancher/v1"
+	v1 "github.com/rancher/shepherd/clients/rancher/v1"
 	"github.com/rancher/shepherd/extensions/clusters"
 	scheme "github.com/rancher/shepherd/pkg/generated/clientset/versioned/scheme"
 	"github.com/sirupsen/logrus"
@@ -53,7 +53,7 @@ func chartActionTestOpts() *PayloadOpts {
 	return &PayloadOpts{
 		InstallOptions: InstallOptions{Cluster: &clusters.ClusterMeta{ID: "c-test-local"}},
 		Name:           "rancher-monitoring",
-		Namespace:      "cattle-monitoring-system",
+		Namespace:      RancherMonitoringNamespace,
 	}
 }
 
@@ -386,7 +386,7 @@ func TestChartActionRequestBuildersDisableClientGoRetries(t *testing.T) {
 
 func TestLogChartActionFailureNilClientIsNoop(t *testing.T) {
 	// Must not panic or log diagnostics when there is no client to read with.
-	LogChartActionFailure(context.Background(), nil, "rancher-charts", "c-local", "cattle-monitoring-system", []string{"rancher-monitoring"})
+	LogChartActionFailure(context.Background(), nil, "rancher-charts", "c-local", RancherMonitoringNamespace, []string{"rancher-monitoring"})
 }
 
 func TestChartOperationStatusForLogRedactsSensitiveFields(t *testing.T) {
@@ -396,7 +396,7 @@ func TestChartOperationStatusForLogRedactsSensitiveFields(t *testing.T) {
 			"chart":       "rancher-monitoring",
 			"version":     "102.0.0",
 			"releaseName": "rancher-monitoring",
-			"namespace":   "cattle-monitoring-system",
+			"namespace":   RancherMonitoringNamespace,
 			"podName":     "helm-operation-abc123",
 			"podCreated":  true,
 			"token":       "kubeconfig-u-abcdef-secrettoken",

actions/charts/longhorn.go

@@ -23,10 +23,14 @@ var (
 	LonghornStaticStorageClass = "longhorn-static"
 	LonghornNamespace          = "longhorn-system"
 	LonghornChartName          = "longhorn"
-	enableDeletionSetting      = map[string]any{
+	LonghornCRDChartName       = "longhorn-crd"
+	longhornChartSettings      = map[string]any{
 		"defaultSettings": map[string]any{
 			"deletingConfirmationFlag": true,
 		},
+		"networkPolicies": map[string]any{
+			"restrictInternalTraffic": false,
+		},
 	}
 )
 
@@ -41,7 +45,7 @@ type LonghornGlobalSettingPut struct {
 // InstallLonghornChart installs the Longhorn chart on the cluster according to data on the payload.
 // Extra values can be passed in through the values argument.
 // This also waits for installation to complete and checks if the deployments are Ready.
-func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values map[string]interface{}) error {
+func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values map[string]any) error {
 	// Callers pass a payload without Name; populate it so chart-action lifecycle and
 	// failure logs carry the chart name.
 	payload.Name = LonghornChartName
@@ -53,21 +57,26 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma
 
 	// If no specific value for setting deletingConfirmationFlag was provided, default to have it enabled so cleanup works as expected.
 	if values == nil {
-		values = enableDeletionSetting
+		values = longhornChartSettings
 	} else {
 		defaultSettings, ok := values["defaultSettings"].(map[string]any)
 		if !ok {
 			return errors.New(`Provided values map has invalid value for "defaultSettings"`)
 		}
 
-		_, ok = defaultSettings["deletingConfirmationFlag"]
+		defaultSettings["deletingConfirmationFlag"] = true
+
+		networkPolicies, ok := values["networkPolicies"].(map[string]any)
 		if !ok {
-			defaultSettings["deletingConfirmationFlag"] = true
+			networkPolicies = map[string]any{}
+			values["networkPolicies"] = networkPolicies
 		}
+
+		networkPolicies["restrictInternalTraffic"] = false
 	}
 
 	chartInstalls := []types.ChartInstall{
-		*NewChartInstall(LonghornChartName+"-crd", payload.Version, payload.Cluster.ID, payload.Cluster.Name, payload.Host, catalog.RancherChartRepo, payload.ProjectID, payload.DefaultRegistry, nil),
+		*NewChartInstall(LonghornCRDChartName, payload.Version, payload.Cluster.ID, payload.Cluster.Name, payload.Host, catalog.RancherChartRepo, payload.ProjectID, payload.DefaultRegistry, nil),
 		*NewChartInstall(LonghornChartName, payload.Version, payload.Cluster.ID, payload.Cluster.Name, payload.Host, catalog.RancherChartRepo, payload.ProjectID, payload.DefaultRegistry, values),
 	}
 
@@ -77,7 +86,7 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma
 	if err != nil {
 		return err
 	}
-	err = ChartActionWithRetry(context.TODO(), client, verbInstall, &payload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbInstall, bodyBytes))
+	err = ChartActionWithRetry(context.TODO(), client, verbInstall, &payload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornCRDChartName}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbInstall, bodyBytes))
 	if err != nil {
 		return err
 	}
@@ -108,10 +117,10 @@ func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID
 	}
 
 	uninstallPayload := &PayloadOpts{
-		InstallOptions: InstallOptions{Cluster: &clusters.ClusterMeta{ID: clusterID}},
-		Name:           LonghornChartName,
-		Namespace:      namespace,
-		Host:           rancherHostname,
+		Cluster:   &clusters.ClusterMeta{ID: clusterID},
+		Name:      LonghornChartName,
+		Namespace: namespace,
+		Host:      rancherHostname,
 	}
 
 	bodyBytes, err := json.Marshal(NewChartUninstallAction())
@@ -133,12 +142,12 @@ func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID
 	if err != nil {
 		return err
 	}
-	err = ChartActionWithRetry(context.TODO(), client, verbUninstall, uninstallPayload, "", []string{LonghornChartName + "-crd"}, buildAppUninstallRequest(catalogClient, namespace, LonghornChartName+"-crd", crdBodyBytes))
+	err = ChartActionWithRetry(context.TODO(), client, verbUninstall, uninstallPayload, "", []string{LonghornCRDChartName}, buildAppUninstallRequest(catalogClient, namespace, LonghornCRDChartName, crdBodyBytes))
 	if err != nil {
 		return err
 	}
 
-	err = waitUninstallation(catalogClient, namespace, LonghornChartName+"-crd")
+	err = waitUninstallation(catalogClient, namespace, LonghornCRDChartName)
 	if err != nil {
 		return err
 	}

actions/charts/ranchermonitoring.go

@@ -9,6 +9,7 @@ import (
 	catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1"
 	"github.com/rancher/shepherd/clients/rancher"
 	"github.com/rancher/shepherd/clients/rancher/catalog"
+	shepherdCharts "github.com/rancher/shepherd/extensions/charts"
 	"github.com/rancher/shepherd/extensions/clusters"
 	"github.com/rancher/shepherd/extensions/defaults"
 	"github.com/rancher/shepherd/extensions/kubectl"
@@ -160,7 +161,21 @@ func InstallRancherMonitoringChart(client *rancher.Client, installOptions *Insta
 	if err != nil {
 		return err
 	}
-	return nil
+
+	// Wait for the monitoring workloads to be Ready so the chart is actually serving
+	// (Prometheus and Alertmanager run as StatefulSets; the operator and Grafana as Deployments).
+	// Without this, callers can race against Prometheus not yet scraping any targets.
+	err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, RancherMonitoringNamespace, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	err = shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, RancherMonitoringNamespace, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	return shepherdCharts.WatchAndWaitStatefulSets(client, installOptions.Cluster.ID, RancherMonitoringNamespace, metav1.ListOptions{})
 }
 
 // newMonitoringChartInstallAction is a private helper function that returns chart install action with monitoring and payload options.

actions/charts/repo.go

@@ -0,0 +1,48 @@
+package charts
+
+import (
+	"context"
+	"time"
+
+	"github.com/rancher/shepherd/clients/rancher"
+	"github.com/rancher/shepherd/clients/rancher/catalog"
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kwait "k8s.io/apimachinery/pkg/util/wait"
+)
+
+// UpdateClusterRepo refreshes the indicated cluster, ensuring that the newest versions are available.
+// Doesn't return and error and logs a warning if updating the cluster fails.
+func UpdateClusterRepo(adminClient *rancher.Client, repoName string) {
+	err := updateClusterRepo(adminClient.Catalog, repoName)
+
+	if err != nil {
+		logrus.Warnf("Failed to refresh cluster repo %s: %s", repoName, err.Error())
+	} else {
+		logrus.Infof("Successfully refreshed cluster repo %s", repoName)
+	}
+}
+
+func updateClusterRepo(catalogClient *catalog.Client, repoName string) error {
+	clusterRepo, err := catalogClient.ClusterRepos().Get(context.Background(), repoName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	forceUpdate := metav1.Now()
+	clusterRepo.Spec.ForceUpdate = &forceUpdate
+
+	_, err = catalogClient.ClusterRepos().Update(context.Background(), clusterRepo, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return kwait.PollUntilContextTimeout(context.Background(), 500*time.Millisecond, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		repo, err := catalogClient.ClusterRepos().Get(ctx, repoName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		return !repo.Status.DownloadTime.Before(&forceUpdate), nil
+	})
+}

actions/kubeapi/longhorn/longhorn.go

@@ -53,10 +53,8 @@ func GetNumberOfLonghornVolumes(client *rancher.Client, clusterID string) (int,
 func CreateS3LonghornBackupTarget(client *rancher.Client, clusterID string, awsCreds cloudcredentials.AmazonEC2CredentialConfig, region, bucketName string) (*longhorn.BackupTarget, error) {
 	secretName := namegenerator.AppendRandomString("aws-creds")
 	secret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      secretName,
-			Namespace: charts.LonghornNamespace,
-		},
+		Name:      secretName,
+		Namespace: charts.LonghornNamespace,
 		StringData: map[string]string{
 			"AWS_ACCESS_KEY_ID":     awsCreds.AccessKey,
 			"AWS_SECRET_ACCESS_KEY": awsCreds.SecretKey,
@@ -164,14 +162,10 @@ func CreateLonghornVolumeBackup(client *rancher.Client, clusterID string, namesp
 	}
 
 	snapshot := longhorn.Snapshot{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       longhornTypes.LonghornKindSnapshot,
-			APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version,
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      namegenerator.AppendRandomString("backup-test"),
-			Namespace: namespace,
-		},
+		Kind:       longhornTypes.LonghornKindSnapshot,
+		APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version,
+		Name:       namegenerator.AppendRandomString("backup-test"),
+		Namespace:  namespace,
 		Spec: longhorn.SnapshotSpec{
 			Volume:         volumeName,
 			CreateSnapshot: true,
@@ -196,16 +190,12 @@ func CreateLonghornVolumeBackup(client *rancher.Client, clusterID string, namesp
 	}
 
 	backup := &longhorn.Backup{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       longhornTypes.LonghornKindBackup,
-			APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version,
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      namegenerator.AppendRandomString("backup-test"),
-			Namespace: namespace,
-			Labels: map[string]string{
-				"backup-volume": volumeName,
-			},
+		Kind:       longhornTypes.LonghornKindBackup,
+		APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version,
+		Name:       namegenerator.AppendRandomString("backup-test"),
+		Namespace:  namespace,
+		Labels: map[string]string{
+			"backup-volume": volumeName,
 		},
 		Spec: longhorn.BackupSpec{
 			SnapshotName: snapshot.Name,
@@ -219,7 +209,7 @@ func CreateLonghornVolumeBackup(client *rancher.Client, clusterID string, namesp
 	}
 
 	client.Session.RegisterCleanupFunc(func() error {
-		return backupController.Client().Delete(context.Background(), namespace, snapshot.Name, metav1.DeleteOptions{})
+		return backupController.Client().Delete(context.Background(), namespace, backup.Name, metav1.DeleteOptions{})
 	})
 
 	err = wait.PollUntilContextTimeout(context.Background(), defaults.FiveHundredMillisecondTimeout, defaults.FiveMinuteTimeout, false, func(context.Context) (bool, error) {
@@ -273,7 +263,7 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b
 	}
 
 	backupVolumeSelector := labels.SelectorFromSet(longhornTypes.GetBackupVolumeWithBackupTargetLabels(backup.Status.BackupTargetName, backup.Status.VolumeName))
-	err = wait.PollUntilContextTimeout(context.Background(), defaults.FiveSecondTimeout, defaults.FiveMinuteTimeout, true, func(context.Context) (bool, error) {
+	err = wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, defaults.FifteenMinuteTimeout, true, func(context.Context) (bool, error) {
 		var backupVolumes longhorn.BackupVolumeList
 		err := backupVolumeController.Client().List(context.Background(), backup.Namespace, &backupVolumes, metav1.ListOptions{LabelSelector: backupVolumeSelector.String()})
 		if err != nil {
@@ -286,17 +276,18 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b
 		return fmt.Errorf("backup volume for volume %s on backup target %s was not created: %w", backup.Status.VolumeName, backup.Status.BackupTargetName, err)
 	}
 
+	// Backups of encrypted volumes must be restored into encrypted volumes. Longhorn reports the backup size without
+	// the LUKS header and adds it back to the backend size of encrypted volumes, so the size can be used as is.
+	encrypted := backup.Status.Labels[longhornTypes.LonghornLabelVolumeEncrypted] == longhornTypes.LonghornLabelValueEnabled
+
 	// This volume is based on the docs: https://longhorn.io/docs/1.11.1/snapshots-and-backups/backup-and-restore/restore-from-a-backup/
 	volume := &longhorn.Volume{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       longhornTypes.LonghornKindVolume,
-			APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version,
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      namegenerator.AppendRandomString("restored-" + backup.Status.VolumeName),
-			Namespace: backup.Namespace,
-		},
+		Kind:       longhornTypes.LonghornKindVolume,
+		APIVersion: longhorn.SchemeGroupVersion.Group + "/" + longhorn.SchemeGroupVersion.Version,
+		Name:       namegenerator.AppendRandomString("restored-" + backup.Status.VolumeName),
+		Namespace:  backup.Namespace,
 		Spec: longhorn.VolumeSpec{
+			Encrypted:  encrypted,
 			Size:       int64(volumeSize),
 			FromBackup: backup.Status.URL,
 			Frontend:   "blockdev",
@@ -319,10 +310,19 @@ func RestoreLonghornVolumeFromBackup(client *rancher.Client, clusterID string, b
 			return false, err
 		}
 
+		for _, condition := range volume.Status.Conditions {
+			if condition.Type == longhorn.VolumeConditionTypeRestore && condition.Reason == longhorn.VolumeConditionReasonRestoreFailure {
+				return false, fmt.Errorf("restore of volume %s from backup %s failed: %s", volume.Name, backup.Name, condition.Message)
+			}
+		}
+
 		return !volume.Status.RestoreRequired && volume.Status.State == longhorn.VolumeStateDetached, nil
 	})
+	if err != nil {
+		return fmt.Errorf("volume %s restored from backup %s did not become ready (state: %s, restoreRequired: %t): %w", volume.Name, backup.Name, volume.Status.State, volume.Status.RestoreRequired, err)
+	}
 
-	return err
+	return nil
 }
 
 // DeleteLonghornVolume deletes the specified volume.longhorn.io resource.

validation/charts/alerting_test.go

@@ -52,31 +52,40 @@ func (a *AlertingTestSuite) SetupSuite() {
 }
 
 func (a *AlertingTestSuite) TestAlertingInstallation() {
-	latestAlertingVersion, err := a.client.Catalog.GetLatestChartVersion(
-		charts.RancherAlertingName,
-		catalog.RancherChartRepo,
-	)
+	alertingChart, err := shepherdCharts.GetChartStatus(a.client, a.cluster.ID, charts.RancherAlertingNamespace, charts.RancherAlertingName)
 	require.NoError(a.T(), err)
 
-	installOptions := &charts.InstallOptions{
-		Cluster:   a.cluster,
-		Version:   latestAlertingVersion,
-		ProjectID: a.project.ID,
-	}
+	if !alertingChart.IsAlreadyInstalled {
+		a.T().Skip("Alerting chart is already installed")
 
-	featureOptions := &charts.RancherAlertingOpts{
-		SMS:   true,
-		Teams: false,
-	}
+		latestAlertingVersion, err := a.client.Catalog.GetLatestChartVersion(
+			charts.RancherAlertingName,
+			catalog.RancherChartRepo,
+		)
+		require.NoError(a.T(), err)
 
-	a.T().Logf(
-		"Installing Rancher Alerting chart on cluster [%s] with version [%s]",
-		a.cluster.Name,
-		latestAlertingVersion,
-	)
+		charts.UpdateClusterRepo(a.client, catalog.RancherChartRepo)
 
-	err = charts.InstallRancherAlertingChart(a.client, installOptions, featureOptions)
-	require.NoError(a.T(), err)
+		installOptions := &charts.InstallOptions{
+			Cluster:   a.cluster,
+			Version:   latestAlertingVersion,
+			ProjectID: a.project.ID,
+		}
+
+		featureOptions := &charts.RancherAlertingOpts{
+			SMS:   true,
+			Teams: false,
+		}
+
+		a.T().Logf(
+			"Installing Rancher Alerting chart on cluster [%s] with version [%s]",
+			a.cluster.Name,
+			latestAlertingVersion,
+		)
+
+		err = charts.InstallRancherAlertingChart(a.client, installOptions, featureOptions)
+		require.NoError(a.T(), err)
+	}
 
 	a.T().Logf(
 		"Waiting for alerting deployments to become ready in namespace [%s]",