diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 89c8640cb3..c1191e3859 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" shepherdCharts "github.com/rancher/shepherd/extensions/charts" @@ -100,6 +101,123 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma return err } +// UpgradeLonghornChart upgrades the longhorn and longhorn-crd charts to +// installOptions.Version on installOptions.Cluster and waits for Deployed state. +// No session cleanup is registered: the uninstall cleanup belongs to the install +// call of the same session, and upgrade-test sessions must not delete the chart. +func UpgradeLonghornChart(client *rancher.Client, installOptions *InstallOptions) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID) + if err != nil { + return err + } + + longhornChartUpgradeActionPayload := &PayloadOpts{ + InstallOptions: *installOptions, + Name: LonghornChartName, + Namespace: LonghornNamespace, + Host: serverSetting.Value, + DefaultRegistry: registrySetting.Value, + } + + chartUpgradeAction := newLonghornChartUpgradeAction(longhornChartUpgradeActionPayload) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + bodyBytes, err := json.Marshal(chartUpgradeAction) + if err != nil { + return err + } + err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, longhornChartUpgradeActionPayload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes)) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + watchAppInterface, err := adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) +} + +// newLonghornChartUpgradeAction is a private helper function that returns the chart upgrade action for Longhorn. +func newLonghornChartUpgradeAction(p *PayloadOpts) *types.ChartUpgradeAction { + // Pass the deletingConfirmationFlag value on the main chart so the upgrade does + // not reset it; the CRD chart takes no Longhorn settings (same split as install). + chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, enableDeletionSetting) + chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil) + chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade} + + chartUpgradeAction := NewChartUpgradeAction(p.Namespace, chartUpgrades) + // Disable OpenAPI validation to avoid schema-conflict failures on the large + // Longhorn CRDs, which frequently change shape between versions. + chartUpgradeAction.DisableOpenAPIValidation = true + + return chartUpgradeAction +} + // UninstallLonghornChart removes Longhorn from the cluster related to the received catalog client object. func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID string, rancherHostname string) error { catalogClient, err := client.GetClusterCatalogClient(clusterID) diff --git a/actions/clusters/local.go b/actions/clusters/local.go index 206e523b81..16940dfd53 100644 --- a/actions/clusters/local.go +++ b/actions/clusters/local.go @@ -1,13 +1,18 @@ package clusters import ( + "context" "errors" "fmt" "regexp" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -37,6 +42,41 @@ func IsRancherVersionAbove(client *rancher.Client, minVersion string) (bool, err return serverVersion.GreaterThan(minimumVersion), nil } +// WaitRancherVersion polls the management server-version setting until it equals +// targetVersion (semver-compared) or timeout elapses. Gates post-upgrade assertions +// on Rancher server convergence; transient read errors during the upgrade rollout +// are retried, not returned. +func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout time.Duration) error { + target, err := parseRancherVersion(targetVersion) + if err != nil { + return err + } + + return wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, timeout, true, func(ctx context.Context) (done bool, err error) { + serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting) + if err != nil { + // The server may be mid-restart during the upgrade rollout; retry the read. + // Logged so a terminal cause (e.g. a revoked token or unreachable API) + // is visible in the log instead of surfacing only as the final timeout. + logrus.Debugf("server-version read failed, retrying: %v", err) + return false, nil + } + + serverVersion, err := parseRancherVersion(serverVersionRawValue.Value) + if err != nil { + logrus.Debugf("server-version value %q not parseable yet, retrying: %v", serverVersionRawValue.Value, err) + return false, nil + } + + if serverVersion.Equal(target) { + return true, nil + } + + logrus.Debugf("Rancher server version is %q, waiting for %q", serverVersionRawValue.Value, targetVersion) + return false, nil + }) +} + func parseRancherVersion(versionInput string) (*semver.Version, error) { trimmedVersion := strings.TrimSpace(strings.TrimPrefix(versionInput, "v")) if trimmedVersion == "" { diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go new file mode 100644 index 0000000000..e7da7d5b29 --- /dev/null +++ b/actions/storage/checksum.go @@ -0,0 +1,221 @@ +package storage + +import ( + "context" + "crypto/sha256" + "fmt" + "regexp" + "strings" + + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/extensions/defaults/namespaces" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + "github.com/rancher/shepherd/extensions/kubeapi/cluster" + "github.com/rancher/shepherd/extensions/kubeconfig" + "github.com/rancher/tests/actions/kubeapi/volumes/persistentvolumeclaims" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" +) + +// ChecksumDataFile is the filename written under MountPath by CreateChecksumWorkload. +const ChecksumDataFile = "checksum-data" + +// checksumContentPattern constrains the content accepted by CreateChecksumWorkload: +// content is interpolated into a single-quoted sh -c command, so a value outside +// alphanumeric/hyphen (notably a quote) would break the quoting. Enforced in code +// so a future caller cannot silently inject shell. +var checksumContentPattern = regexp.MustCompile(`^[A-Za-z0-9-]+$`) + +// CreateChecksumWorkload creates a 1GiB PVC named on storageClassName in the +// default namespace, mounts it into a single-replica nginx deployment named at +// MountPath, waits for rollout, then writes content to MountPath/ChecksumDataFile via +// kubectl exec. Deterministic names let a later test phase (running as a separate +// process) relocate the resources. content must be shell-safe (alphanumeric/hyphen); +// it is interpolated into an sh -c command. No session cleanup is registered: callers +// are upgrade tests whose resources must outlive the process. +func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageClassName string, name string, content string) error { + if !checksumContentPattern.MatchString(content) { + return fmt.Errorf("content %q must match %s: it is interpolated into a single-quoted sh -c command", + content, checksumContentPattern.String()) + } + + storageClass, err := GetStorageClass(client, clusterID, storageClassName) + if err != nil { + return err + } + + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + accessModes := []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + } + + // Deliberate deviation from CreatePVC: no PV-delete/Longhorn-volume cleanup funcs + // are registered, because these resources must survive into later upgrade phases. + _, err = persistentvolumeclaims.CreatePersistentVolumeClaim( + client, + clusterID, + name, + "test-pvc-volume", + namespaces.Default, + 1, + accessModes, + nil, + &storageClass, + ) + if err != nil { + return err + } + + pvc := &corev1.PersistentVolumeClaim{} + // TenMinuteTimeout: the first volume on a freshly installed Longhorn waits + // on engine-image pull, instance-manager startup, and CSI registration, + // which routinely exceeds a minute on a cold cluster. + err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.TenMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + pvc, err = wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pvc.Status.Phase == persistentvolumeclaims.PersistentVolumeBoundStatus, nil + }) + if err != nil { + return err + } + logrus.Debugf("PVC %s is Bound to volume %s", pvc.Name, pvc.Spec.VolumeName) + + // Shared nginx+PVC deployment construction with CreatePVCWorkload; the + // deterministic name makes PVC, volume mount, deployment, and relocation + // label all equal name, so later phases re-find the workload from a + // separate process. + _, err = createNginxDeploymentWithPVC(client, clusterID, name, name, name) + if err != nil { + return err + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + writeCommand := []string{"sh", "-c", "printf '%s' '" + content + "' > " + MountPath + "/" + ChecksumDataFile} + logrus.Debugf("Writing %s/%s on pod %s", MountPath, ChecksumDataFile, pod.Name) + _, err = kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, writeCommand) + + return err +} + +// VerifyChecksumWorkload relocates the PVC and pod created by CreateChecksumWorkload, +// asserts the PVC is still Bound, and asserts the sha256 of MountPath/ChecksumDataFile +// inside the running pod equals sha256(content). +func VerifyChecksumWorkload(client *rancher.Client, clusterID string, name string, content string) error { + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + pvc, err := wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return err + } + + if pvc.Status.Phase != persistentvolumeclaims.PersistentVolumeBoundStatus { + return fmt.Errorf("PVC %s is %s, expected %s", name, pvc.Status.Phase, persistentvolumeclaims.PersistentVolumeBoundStatus) + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + checksumCommand := []string{"sha256sum", MountPath + "/" + ChecksumDataFile} + output, err := kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, checksumCommand) + if err != nil { + return err + } + + // The exec runs with a TTY, so the output carries \r line endings; strip them + // before extracting the hash (first whitespace-separated field). + outputFields := strings.Fields(strings.ReplaceAll(output.String(), "\r", "")) + if len(outputFields) == 0 { + return fmt.Errorf("empty sha256sum output on pod %s", pod.Name) + } + + actualChecksum := outputFields[0] + expectedChecksum := fmt.Sprintf("%x", sha256.Sum256([]byte(content))) + if actualChecksum != expectedChecksum { + return fmt.Errorf("checksum mismatch on pod %s: expected %s, got %s", pod.Name, expectedChecksum, actualChecksum) + } + + return nil +} + +// getChecksumPod returns the pod in the default namespace labeled with +// DeploymentIdentifierLabel == name, waiting until it exists and is Running. +func getChecksumPod(client *rancher.Client, clusterID string, name string) (*steveV1.SteveAPIObject, error) { + var checksumPod *steveV1.SteveAPIObject + err := wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, defaults.TwoMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + steveclient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return false, err + } + + podsResp, err := steveclient.SteveType(stevetypes.Pod).NamespacedSteveClient(namespaces.Default).List(nil) + if err != nil { + return false, err + } + + for i := range podsResp.Data { + pod := &podsResp.Data[i] + if pod.Labels[DeploymentIdentifierLabel] != name { + continue + } + + podStatus := &corev1.PodStatus{} + err = steveV1.ConvertToK8sType(pod.Status, podStatus) + if err != nil { + return false, err + } + + if podStatus.Phase != corev1.PodRunning { + logrus.Debugf("Pod %s is %s, waiting for Running", pod.Name, podStatus.Phase) + return false, nil + } + + checksumPod = pod + return true, nil + } + + return false, nil + }) + + return checksumPod, err +} + +// getDownstreamRestConfig builds a rest config for kubectl exec against the cluster. +func getDownstreamRestConfig(client *rancher.Client, clusterID string) (*rest.Config, error) { + kubeConfig, err := kubeconfig.GetKubeconfig(client, clusterID) + if err != nil { + return nil, err + } + + return (*kubeConfig).ClientConfig() +} diff --git a/actions/storage/pvc.go b/actions/storage/pvc.go index 163c0df025..4030e6823d 100644 --- a/actions/storage/pvc.go +++ b/actions/storage/pvc.go @@ -158,7 +158,6 @@ func CreatePVC(client *rancher.Client, clusterID string, storageClassName string } return false, err }) - if err != nil { return nil, "", err } @@ -185,19 +184,25 @@ func CreatePVCWorkload(t *testing.T, client *rancher.Client, clusterID string, s persistentVolumeClaim, volumeName, err := CreatePVC(client, clusterID, storageClassName) require.NoError(t, err) - logrus.Debugf("Creating Nginx deployment with %s PVC (volume name %s)", persistentVolumeClaim.Name, volumeName) - nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName) + // Random workload name: this helper targets single-process functional tests; + // callers needing a deterministic (cross-phase relocatable) name call + // createNginxDeploymentWithPVC directly (see CreateChecksumWorkload). + deploymentName := namegenerator.AppendRandomString(containerNamePrefix) + logrus.Debugf("Creating Nginx deployment %s with %s PVC (volume name %s)", deploymentName, persistentVolumeClaim.Name, volumeName) + nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName, deploymentName) require.NoError(t, err) return nginxResponse } -// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace -func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName string) (*steveV1.SteveAPIObject, error) { +// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace. +// The deployment and its DeploymentIdentifierLabel both use deploymentName, letting callers +// pick a random name (functional tests) or a deterministic name (workloads a later test +// phase must re-find from a separate process). +func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName, deploymentName string) (*steveV1.SteveAPIObject, error) { logrus.Tracef("Vol: %s", volName) logrus.Tracef("Pod: %s", pvcName) - containerName := namegenerator.AppendRandomString(containerNamePrefix) volMount := &corev1.VolumeMount{ MountPath: MountPath, Name: volName, @@ -218,8 +223,8 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{*volMount}, []corev1.EnvFromSource{}, nil, nil, nil) - podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: containerName}, nil) - deployment := wloads.NewDeploymentTemplate(containerName, namespaces.Default, podTemplate, true, nil) + podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: deploymentName}, nil) + deployment := wloads.NewDeploymentTemplate(deploymentName, namespaces.Default, podTemplate, true, nil) deploymentResp, err := steveclient.SteveType(stevetypes.Deployment).Create(deployment) if err != nil { @@ -227,7 +232,7 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ - FieldSelector: "metadata.name=" + containerName, + FieldSelector: "metadata.name=" + deploymentName, }) return deploymentResp, err diff --git a/interoperability/charts/constants.go b/interoperability/charts/constants.go index f6f57c44c6..39d7f276e7 100644 --- a/interoperability/charts/constants.go +++ b/interoperability/charts/constants.go @@ -1,12 +1,15 @@ package charts const ( - ExtensionNamespace = "cattle-ui-plugin-system" - HarvesterExtensionName = "harvester" - NeuVectorUIExtensionName = "neuvector-ui-ext" - StackstateExtensionsName = "observability" - UIPluginRepoName = "rancher-ui-plugins" - StackstateK8sAgent = "stackstate-k8s-agent" + ExtensionNamespace = "cattle-ui-plugin-system" + HarvesterExtensionName = "harvester" + NeuVectorUIExtensionName = "neuvector-ui-ext" + StackstateExtensionsName = "observability" + UIPluginRepoName = "rancher-ui-plugins" + // StackstateK8sAgent is the suse-observability-agent chart (renamed + // from the legacy stackstate-k8s-agent partner chart) from the same + // prime server-charts repo as the server chart. + StackstateK8sAgent = "suse-observability-agent" StackstateNamespace = "stackstate" StackstateCRD = "observability.rancher.io.configuration" RancherPartnerChartRepo = "rancher-partner-charts" diff --git a/interoperability/charts/stackstate.go b/interoperability/charts/stackstate.go index 7eebe7a095..c6a45af735 100644 --- a/interoperability/charts/stackstate.go +++ b/interoperability/charts/stackstate.go @@ -3,12 +3,12 @@ package charts import ( "context" "fmt" + "time" catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" rv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" - "github.com/rancher/shepherd/extensions/defaults" "github.com/rancher/shepherd/pkg/api/steve/catalog/types" "github.com/rancher/shepherd/pkg/wait" "github.com/rancher/tests/actions/charts" @@ -16,25 +16,138 @@ import ( "github.com/rancher/tests/interoperability/observability" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kwait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" ) -var ( - timeoutSeconds = int64(defaults.TwoMinuteTimeout) +const ( + // clusterRepoCreateRetries bounds the create attempts for a ClusterRepo on + // a freshly settled management cluster. + clusterRepoCreateRetries = 4 + // clusterRepoCreateRetryDelay is the spacing between ClusterRepo create attempts. + clusterRepoCreateRetryDelay = 15 * time.Second + + // chartActionRetryAttempts bounds the attempts for a chart install or + // upgrade action POST. + chartActionRetryAttempts = 3 + // chartActionRetryDelay is the spacing between chart action attempts, + // giving the management cluster time to finish and cache a large chart's + // first render. + chartActionRetryDelay = 30 * time.Second + + // stackstateHelmActionTimeout is the helm timeout carried by the chart + // actions. The default ten minutes does not cover a cold cluster pulling + // the full SUSE Observability image set: build 35's install hit the helm + // deadline with workloads still pulling, and the app failed. Forty-five + // minutes covers cold pulls with room to spare while keeping a + // deadline-exceeded failure, which dumps its diagnostics, inside the + // pipeline's per-phase test timeout. + stackstateHelmActionTimeout = 45 * time.Minute + // chartTemplatePollInterval and chartTemplatePollTimeout bound the wait + // for a fresh ClusterRepo's chart template to be fetched and cached + // server-side; the template requires downloading the chart tarball, which + // can span several request windows for large prime charts. + chartTemplatePollInterval = 10 * time.Second + chartTemplatePollTimeout = 10 * time.Minute ) +// appWatchSeconds bounds the app watches: it must exceed the helm action +// timeout comfortably, or the watch gives up on an install that is still +// running healthily (build 36's watch tied with the thirty-minute helm +// deadline and lost by seconds). +var appWatchSeconds = int64((stackstateHelmActionTimeout + 10*time.Minute).Seconds()) + +// WaitForChartTemplate polls the chart template info link until the catalog +// has fetched and cached the chart version template server-side. Fresh +// ClusterRepos resolve their version list quickly, but the per-version +// template needs the chart tarball, which large prime charts download across +// several request windows; until the template is cached, chart action POSTs +// resolve it synchronously and can outlive the request deadline, answering +// with a bare internal error. +func WaitForChartTemplate(catalogClient *catalog.Client, repoName, chartName, chartVersion string) error { + err := kwait.PollUntilContextTimeout(context.TODO(), chartTemplatePollInterval, chartTemplatePollTimeout, true, func(context.Context) (bool, error) { + _, err := catalogClient.GetChartValues(repoName, chartName, chartVersion) + return err == nil, nil + }) + if err != nil { + return fmt.Errorf("chart template %s %s from repo %s did not become available within %s: %w", chartName, chartVersion, repoName, chartTemplatePollTimeout, err) + } + return nil +} + +// retryChartAction runs a chart install or upgrade action POST with bounded +// retries. The suse-observability charts are large: their first action can +// outlive the Rancher proxy's request timeout while templates render and +// cache, answering with a bare internal error. An attempt that persisted +// server-side despite its error surfaces as an already-exists conflict, which +// satisfies the action. +func retryChartAction(action string, fn func() error) error { + var lastErr error + for attempt := 1; attempt <= chartActionRetryAttempts; attempt++ { + lastErr = fn() + if lastErr == nil { + return nil + } + if k8sErrors.IsAlreadyExists(lastErr) || k8sErrors.IsConflict(lastErr) { + log.Infof("%s already exists after a failed attempt; treating as applied", action) + return nil + } + log.Warnf("%s attempt %d/%d failed: %v", action, attempt, chartActionRetryAttempts, lastErr) + if attempt < chartActionRetryAttempts { + // Spacing between attempts: with immediate=false the first condition + // check runs only after the interval, so this poll acts as the + // context-aware sleep. Its error is ignored because the condition + // succeeds at that first check (the timeout outlives the interval + // and the context is never cancelled). + _ = kwait.PollUntilContextTimeout(context.TODO(), chartActionRetryDelay, chartActionRetryDelay+time.Second, false, func(context.Context) (bool, error) { + return true, nil + }) + } + } + return fmt.Errorf("%s: %w", action, lastErr) +} + +// watchAppUntilState watches an app until it reaches the target state, +// retrying the watch on transient watch-stream drops: helm keeps operating +// server-side, so only the watch is re-established, never the chart action +// (build 44 lost a healthy install to a single dropped stream). StatusFailed +// always errors immediately so failures surface fast. +func watchAppUntilState(catalogClient *catalog.Client, namespace, appName, targetState string) error { + return charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + watchAppInterface, err := catalogClient.Apps(namespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + appName, + TimeoutSeconds: &appWatchSeconds, + }) + if err != nil { + return err + } + + return wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusFailed) { + return false, fmt.Errorf("app %s entered failed state", app.Name) + } + if state == targetState { + return true, nil + } + return false, nil + }) + }) +} + // InstallStackStateServerChart installs the StackState chart into the specified Kubernetes cluster and namespace. // It uses Rancher client and configuration details, and waits for the chart deployment to complete successfully. // Parameters: // - client: the Rancher client used for connecting to the Kubernetes cluster. // - installOptions: the installation options including cluster info and chart version. -// - stackstateConfigs: configuration details for StackState such as the service token and URL. // - systemProjectID: the ID of the system project where the chart is deployed. // - additionalValues: additional Helm chart values as a map for custom configurations. // Returns an error if the chart installation fails or its status cannot be confirmed. func InstallStackStateServerChart(client *rancher.Client, installOptions *charts.InstallOptions, systemProjectID string, additionalValues map[string]interface{}) error { - serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) if err != nil { log.Info("Error getting server setting.") @@ -56,31 +169,14 @@ func InstallStackStateServerChart(client *rancher.Client, installOptions *charts return err } - err = catalogClient.InstallChart(chartInstallAction, StackStateServerChartRepo) - if err != nil { - log.Info("Error installing the StackState chart") - return err - } - - watchAppInterface, err := catalogClient.Apps(StackStateServerNamespace).Watch(context.TODO(), metav1.ListOptions{ - FieldSelector: "metadata.name=" + StackStateServerChartRepo, - TimeoutSeconds: &defaults.WatchTimeoutSeconds, + err = retryChartAction("suse-observability server chart install", func() error { + return catalogClient.InstallChart(chartInstallAction, StackStateServerChartRepo) }) if err != nil { - log.Info("StackState App failed to install") return err } - err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { - app := event.Object.(*catalogv1.App) - - state := app.Status.Summary.State - if state == string(catalogv1.StatusDeployed) { - return true, nil - } - return false, nil - }) - + err = watchAppUntilState(catalogClient, StackStateServerNamespace, StackStateServerChartRepo, string(catalogv1.StatusDeployed)) if err != nil { log.Info("Unable to obtain the status of the installed app ") return err @@ -122,7 +218,7 @@ func InstallStackstateAgentChart(client *rancher.Client, installOptions *charts. watchAppInterface, err := catalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ FieldSelector: "metadata.name=" + StackstateK8sAgent, - TimeoutSeconds: &timeoutSeconds, + TimeoutSeconds: &appWatchSeconds, }) if err != nil { return err @@ -173,7 +269,7 @@ func InstallStackstateAgentChart(client *rancher.Client, installOptions *charts. watchNamespaceInterface, err := adminNamespaceResource.Watch(context.TODO(), metav1.ListOptions{ FieldSelector: "metadata.name=" + StackstateNamespace, - TimeoutSeconds: &timeoutSeconds, + TimeoutSeconds: &appWatchSeconds, }) if err != nil { return err @@ -187,32 +283,15 @@ func InstallStackstateAgentChart(client *rancher.Client, installOptions *charts. }) }) - err = catalogClient.InstallChart(chartInstallAction, RancherPartnerChartRepo) - if err != nil { - log.Info("Errored installing the chart") - return err - } - - // wait for chart to be fully deployed - watchAppInterface, err := catalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ - FieldSelector: "metadata.name=" + StackstateK8sAgent, - TimeoutSeconds: &defaults.WatchTimeoutSeconds, + err = retryChartAction("suse-observability-agent chart install", func() error { + return catalogClient.InstallChart(chartInstallAction, StackStateServerChartRepo) }) if err != nil { - log.Info("Unable to obtain the installed app ") return err } - err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { - app := event.Object.(*catalogv1.App) - - state := app.Status.Summary.State - if state == string(catalogv1.StatusDeployed) { - return true, nil - } - return false, nil - }) - + // wait for chart to be fully deployed + err = watchAppUntilState(catalogClient, StackstateNamespace, StackstateK8sAgent, string(catalogv1.StatusDeployed)) if err != nil { log.Info("Unable to obtain the status of the installed app ") return err @@ -232,20 +311,21 @@ func newStackstateAgentChartInstallAction(p *charts.PayloadOpts, stackstateConfi }, } - chartInstall := charts.NewChartInstall(p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, rancherPartnerCharts, systemProjectID, p.DefaultRegistry, stackstateValues) + chartInstall := charts.NewChartInstall(p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, StackStateServerChartRepo, systemProjectID, p.DefaultRegistry, stackstateValues) chartInstalls := []types.ChartInstall{*chartInstall} chartInstallAction := charts.NewChartInstallAction(p.Namespace, p.ProjectID, chartInstalls) + chartInstallAction.Timeout = &metav1.Duration{Duration: stackstateHelmActionTimeout} return chartInstallAction } func newStackStateServerChartInstallAction(p *charts.PayloadOpts, systemProjectID string, additionalValues map[string]interface{}) *types.ChartInstallAction { - chartInstall := charts.NewChartInstall(p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, StackStateServerChartRepo, systemProjectID, p.DefaultRegistry, additionalValues) chartInstalls := []types.ChartInstall{*chartInstall} chartInstallAction := charts.NewChartInstallAction(p.Namespace, p.ProjectID, chartInstalls) + chartInstallAction.Timeout = &metav1.Duration{Duration: stackstateHelmActionTimeout} return chartInstallAction } @@ -271,7 +351,9 @@ func UpgradeStackstateAgentChart(client *rancher.Client, installOptions *charts. return err } - err = catalogClient.UpgradeChart(chartUpgradeAction, RancherPartnerChartRepo) + err = retryChartAction("suse-observability-agent chart upgrade", func() error { + return catalogClient.UpgradeChart(chartUpgradeAction, StackStateServerChartRepo) + }) if err != nil { return err } @@ -286,45 +368,16 @@ func UpgradeStackstateAgentChart(client *rancher.Client, installOptions *charts. return err } - // wait for chart to be in status pending upgrade - watchAppInterface, err := adminCatalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ - FieldSelector: "metadata.name=" + StackstateK8sAgent, - TimeoutSeconds: &defaults.WatchTimeoutSeconds, - }) - if err != nil { - return err - } - - err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { - app := event.Object.(*catalogv1.App) - - state := app.Status.Summary.State - if state == string(catalogv1.StatusPendingUpgrade) { - return true, nil - } - return false, nil - }) + // Gate on pending-upgrade before waiting for the new deployed state: the + // app starts this flow already deployed, so the deployed watch alone + // returns immediately on the old state (build 45). Pending-upgrade spans + // the whole helm operation, so a re-established watch cannot miss it. + err = watchAppUntilState(adminCatalogClient, StackstateNamespace, StackstateK8sAgent, string(catalogv1.StatusPendingUpgrade)) if err != nil { return err } - watchAppInterface, err = adminCatalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ - FieldSelector: "metadata.name=" + StackstateK8sAgent, - TimeoutSeconds: &defaults.WatchTimeoutSeconds, - }) - if err != nil { - return err - } - - err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { - app := event.Object.(*catalogv1.App) - - state := app.Status.Summary.State - if state == string(catalogv1.StatusDeployed) { - return true, nil - } - return false, nil - }) + err = watchAppUntilState(adminCatalogClient, StackstateNamespace, StackstateK8sAgent, string(catalogv1.StatusDeployed)) if err != nil { return err } @@ -334,7 +387,6 @@ func UpgradeStackstateAgentChart(client *rancher.Client, installOptions *charts. // newStackstateAgentChartUpgradeAction is a private helper function that returns chart upgrade action. func newStackstateAgentChartUpgradeAction(p *charts.PayloadOpts, stackstateConfigs *observability.StackStateConfig) *types.ChartUpgradeAction { - stackstateValues := map[string]interface{}{ "stackstate": map[string]interface{}{ "cluster": map[string]interface{}{ @@ -348,23 +400,156 @@ func newStackstateAgentChartUpgradeAction(p *charts.PayloadOpts, stackstateConfi chartUpgrade := charts.NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, stackstateValues) chartUpgrades := []types.ChartUpgrade{*chartUpgrade} chartUpgradeAction := charts.NewChartUpgradeAction(p.Namespace, chartUpgrades) + chartUpgradeAction.Timeout = &metav1.Duration{Duration: stackstateHelmActionTimeout} + + return chartUpgradeAction +} + +// UninstallStackStateApp removes the app and its helm release from the +// cluster. A partial install leaves the release name taken and helm refuses +// every later install with "cannot re-use a name that is still in use", so +// re-entrant suites uninstall a stale app before installing again. Waits +// until the app object is gone. +func UninstallStackStateApp(client *rancher.Client, clusterID, namespace, appName string) error { + uninstallAction := &types.ChartUninstallAction{ + Timeout: &metav1.Duration{Duration: stackstateHelmActionTimeout}, + } + + catalogClient, err := client.GetClusterCatalogClient(clusterID) + if err != nil { + return err + } + + if err := retryChartAction(fmt.Sprintf("%s chart uninstall", appName), func() error { + return catalogClient.UninstallChart(appName, namespace, uninstallAction) + }); err != nil { + return err + } + + return kwait.PollUntilContextTimeout(context.TODO(), chartTemplatePollInterval, chartTemplatePollTimeout, true, func(context.Context) (bool, error) { + _, err := catalogClient.Apps(namespace).Get(context.TODO(), appName, metav1.GetOptions{}) + return k8sErrors.IsNotFound(err), nil + }) +} + +// UpgradeStackStateServerChart upgrades the suse-observability server chart on the cluster +// carried by installOptions and waits for the app to transition to deployed state. +// additionalValues must carry the install-time server values (base config, ingress and HA +// sizing profile): the chart-upgrade path deliberately re-applies those values instead of +// a blind version bump, because upgrading on chart defaults would silently drop the HA +// sizing profile and the stackstate service credentials (url/api key/license), breaking +// the agent-to-server pipeline mid-upgrade (rancher/tests#946 Further Notes calls this +// per-product values migration out explicitly). Wrap calls in charts.RetryOnWatchError, +// as with UpgradeStackstateAgentChart, to absorb transient watch failures. +func UpgradeStackStateServerChart(client *rancher.Client, installOptions *charts.InstallOptions, additionalValues map[string]interface{}) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + stackstateServerChartUpgradeActionPayload := &charts.PayloadOpts{ + InstallOptions: *installOptions, + Name: StackStateServerChartRepo, + Namespace: StackStateServerNamespace, + Host: serverSetting.Value, + } + + chartUpgradeAction := newStackStateServerChartUpgradeAction(stackstateServerChartUpgradeActionPayload, additionalValues) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + err = retryChartAction("suse-observability server chart upgrade", func() error { + return catalogClient.UpgradeChart(chartUpgradeAction, StackStateServerChartRepo) + }) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + // Gate on pending-upgrade before waiting for the new deployed state: the + // app starts this flow already deployed, so the deployed watch alone + // returns immediately on the old state (build 45). Pending-upgrade spans + // the whole helm operation, so a re-established watch cannot miss it. + err = watchAppUntilState(adminCatalogClient, StackStateServerNamespace, StackStateServerChartRepo, string(catalogv1.StatusPendingUpgrade)) + if err != nil { + return err + } + + err = watchAppUntilState(adminCatalogClient, StackStateServerNamespace, StackStateServerChartRepo, string(catalogv1.StatusDeployed)) + if err != nil { + return err + } + + return nil +} + +// newStackStateServerChartUpgradeAction is a private helper function that returns the chart +// upgrade action carrying the install-time server values, mirroring the agent upgrade action. +func newStackStateServerChartUpgradeAction(p *charts.PayloadOpts, additionalValues map[string]interface{}) *types.ChartUpgradeAction { + chartUpgrade := charts.NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, additionalValues) + chartUpgrades := []types.ChartUpgrade{*chartUpgrade} + chartUpgradeAction := charts.NewChartUpgradeAction(p.Namespace, chartUpgrades) + chartUpgradeAction.Timeout = &metav1.Duration{Duration: stackstateHelmActionTimeout} return chartUpgradeAction } // CreateClusterRepo creates a new ClusterRepo resource in the Kubernetes cluster using the provided catalog client. // It takes the client, repository name, and repository URL as arguments and returns an error if the operation fails. +// Creation is retried a bounded number of times with spacing: freshly deployed +// management clusters have answered ClusterRepo POSTs with bare internal errors +// while their catalog controllers and validators were still settling. func CreateClusterRepo(client *rancher.Client, catalogClient *catalog.Client, name, url string) error { ctx := context.Background() repo := &rv1.ClusterRepo{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: rv1.RepoSpec{URL: url}, } - _, err := catalogClient.ClusterRepos().Create(ctx, repo, metav1.CreateOptions{}) - client.Session.RegisterCleanupFunc(func() error { + var lastErr error + for attempt := 1; attempt <= clusterRepoCreateRetries; attempt++ { + _, lastErr = catalogClient.ClusterRepos().Create(ctx, repo, metav1.CreateOptions{}) + if lastErr == nil { + break + } + // A failed attempt may still have persisted server-side; a present + // repo satisfies the caller's desired state, so treat AlreadyExists + // as success and fall through to the cleanup registration. + if k8sErrors.IsAlreadyExists(lastErr) { + log.Infof("ClusterRepo %s already exists after a failed create attempt; treating as created", name) + lastErr = nil + break + } + log.Warnf("ClusterRepo %s create attempt %d/%d failed: %v", name, attempt, clusterRepoCreateRetries, lastErr) + if attempt < clusterRepoCreateRetries { + // Spacing between attempts: with immediate=false the first condition + // check runs only after the interval, so this poll acts as the + // context-aware sleep. Its error is ignored because the condition + // succeeds at that first check (the timeout outlives the interval + // and the context is never cancelled). + _ = kwait.PollUntilContextTimeout(ctx, clusterRepoCreateRetryDelay, clusterRepoCreateRetryDelay+time.Second, false, func(context.Context) (bool, error) { + return true, nil + }) + } + } + if lastErr != nil { + return fmt.Errorf("creating ClusterRepo %s at %s: %w", name, url, lastErr) + } - var propagation = metav1.DeletePropagationForeground + client.Session.RegisterCleanupFunc(func() error { + propagation := metav1.DeletePropagationForeground err := catalogClient.ClusterRepos().Delete(context.Background(), name, metav1.DeleteOptions{PropagationPolicy: &propagation}) if err != nil { return err @@ -372,7 +557,7 @@ func CreateClusterRepo(client *rancher.Client, catalogClient *catalog.Client, na return err }) - return err + return nil } // StructToMap Helper function to convert struct to map[string]interface{} diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e new file mode 100755 index 0000000000..7b3b6ac24d --- /dev/null +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -0,0 +1,1067 @@ +#!groovy +/** + * Dedicated Rancher server upgrade validation pipeline (rancher/tests#968, PRD #946). + * + * Six-stage flow, community leg: + * 1. Deploys a 2.14-latest environment via qa-infra-automation + * (make backend-s3, workspace-new, infra-up, cluster, rancher) + * 2. Pre-upgrade phase: installs Longhorn + seeds a checksum workload + * (validation/rancherupgrade, TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade) + * 3. Upgrades the Rancher server in place via `make rancher-upgrade` + * and re-injects the rotated admin token into the test config + * 4. Post-upgrade phase: asserts server version, Longhorn survival, workload data + * 5. Chart-upgrade phase: upgrades Longhorn to the version offered by the + * upgraded Rancher's catalog and re-verifies the workload data + * 6. Always destroys the environment (infra-down AUTO_APPROVE=yes) + * + * Versions resolve at runtime when RANCHER_VERSION_FROM / RANCHER_VERSION_TO_UPGRADE + * are empty (qa-infra-automation `make rancher-resolve-version`); pins override. + * + * Structurally cloned from validation/Jenkinsfile.e2e (deploy flow, admin-token + * injection, Qase testrun lifecycle, destroy post) and validation/pipeline/ + * Jenkinsfile.airgap-rke2-tests (make.runTarget call shapes, teardown guard). + * Unlike Jenkinsfile.e2e, this pipeline has no parallel downstream fan-out: the + * three test phases run sequentially against one shared Qase test run because + * each phase depends on the previous one's in-cluster state. + * + * Consumes shared functions from qa-jenkins-library: + * make.runTarget (backend-s3, workspace-new, infra-up, cluster, rancher, + * rancher-resolve-version, rancher-upgrade, infra-down) + * airgap.standardCheckout + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig, + * infrastructure.generateWorkspaceName, infrastructure.archiveWorkspaceName, + * infrastructure.writeSshKey, infrastructure.detectPublicIp + * property.useWithProperties + * + * Admin token generation uses a direct ansible-playbook call inside the + * rancher-go-test image (no make token target exists upstream yet). + */ + +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +def DEFAULT_RANCHER_PASSWORD = 'X%S3bPyg@n8kWcRs675l#fH0' + +// Build-scoped image tags: every direct docker invocation below runs the +// pinned tag so this job never depends on (or mutates) whatever some other +// job last built under the shared :latest tag on the same agent. The :latest +// alias is still applied at build time because qa-jenkins-library's +// make.runTarget resolves that name for its own container runs. +def INFRA_TOOLS_IMAGE = "rancher-infra-tools:pitupgrade-${env.BUILD_NUMBER}" +def GO_TEST_IMAGE = "rancher-go-test:pitupgrade-${env.BUILD_NUMBER}" + +// Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. +// Each phase is its own go test invocation (-run regex), so cross-phase state +// lives in-cluster under deterministic names, never in process memory. +def PRE_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade"' +def POST_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade"' +def CHART_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade"' + +/** + * Shared teardown helper: re-renders tfvars from params, tears down tofu workspace + * via make infra-down AUTO_APPROVE=yes. + * Sets env.INFRA_CLEANED to 'true' on success. + */ +def destroyInfrastructure(String reason) { + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" + } + + if (!wsName || !env.INFRA_DIR || env.INFRA_CLEANED == 'true') { + echo "Skipping teardown (${reason}): no workspace, no infra dir, or already cleaned" + return + } + + try { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + // Re-render terraform.tfvars so tofu destroy can access AWS state + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + make.runTarget(target: 'infra-down', dir: env.INFRA_DIR, makeArgs: "WORKSPACE=${wsName} AUTO_APPROVE=yes") + env.INFRA_CLEANED = 'true' + } + } catch (cleanupErr) { + echo "Teardown failed (${reason}): ${cleanupErr.message}" + } +} + +/** + * Failure-only diagnostics: capture cluster state through the rancher-infra-tools + * container (kubectl + helm + kubeconfig from the deploy) into workspace files + * and archive them. Best-effort; never masks the failure it reports on. + */ +def collectUpgradeDiagnostics() { + if (!env.INFRA_DIR) { + echo 'Skipping diagnostics: no infra checkout available' + return + } + try { + def workspace = pwd() + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -e KUBECONFIG=/workspace/${env.INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml \ + -w /workspace \ + ${INFRA_TOOLS_IMAGE} \ + sh -c ' + set +e + kubectl get nodes -o wide > diagnostics-nodes.txt 2>&1 + kubectl get pods -A > diagnostics-pods.txt 2>&1 + helm history rancher -n cattle-system > diagnostics-helm-history.txt 2>&1 + kubectl logs -n cattle-system -l app=rancher --tail=300 > diagnostics-rancher-logs.txt 2>&1 + kubectl logs -n cattle-cluster-system -l app=cattle-cluster-agent --tail=300 > diagnostics-agent-logs.txt 2>&1 + exit 0 + ' + """ + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'diagnostics-*.txt', allowEmptyArchive: true + } + } catch (diagErr) { + echo "Diagnostics collection failed (non-fatal): ${diagErr.message}" + } +} + +pipeline { + agent any + + options { + ansiColor('xterm') + // Budget: three sequential test phases (TEST_TIMEOUT each, default 60m) + // plus deploy, server upgrade, and destroy (~90m overhead). Must exceed + // the sum of the stage budgets or a legitimately slow run aborts + // mid-phase; revisit when raising TEST_TIMEOUT. + timeout(time: 300, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + // ── Repository parameters ────────────────────────────── + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + // Repo params keep whatever the job was last triggered with: a + // declarative pipeline re-applies its parameters{} block to the job + // after each run, and hard-coded defaults here would clobber the + // JJB-configured per-controller values (fork + slice branches on the + // staging controller, upstream main on the main controller). + string( + name: 'TESTS_REPO_URL', + defaultValue: params.TESTS_REPO_URL ?: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: params.TESTS_BRANCH ?: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: params.QA_INFRA_REPO_URL ?: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: params.QA_INFRA_BRANCH ?: 'main', + description: 'Branch of qa-infra-automation repository' + ) + + // ── Upgrade parameters ───────────────────────────────── + string( + name: 'RANCHER_VERSION_FROM', + defaultValue: '', + description: 'Source Rancher version to deploy (empty = resolve latest 2.14.x patch at runtime)' + ) + string( + name: 'RANCHER_VERSION_TO_UPGRADE', + defaultValue: '', + description: 'Target Rancher version for the server upgrade (empty = resolve latest released final at runtime)' + ) + string( + name: 'RANCHER_CHART_REPO_FLAVOR', + defaultValue: 'community', + description: 'Chart repo flavor (community|prime): selects the chart index for version resolution, the repo the pre-upgrade Rancher is deployed from, and the repo the server upgrade pulls from' + ) + + // ── Infrastructure parameters ────────────────────────── + string( + name: 'HOSTNAME_PREFIX', + defaultValue: 'pitupgrade', + description: 'Hostname prefix for *.qa.rancher.space and other AWS resources' + ) + string( + name: 'RKE2_VERSION', + defaultValue: 'v1.35.1+rke2r1', + description: 'RKE2 version to deploy' + ) + password( + name: 'RANCHER_BOOTSTRAP_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher bootstrap password for initial setup' + ) + password( + name: 'RANCHER_ADMIN_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher admin password after bootstrap' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Collect diagnostics and tear down infrastructure if pipeline fails' + ) + booleanParam( + name: 'DESTROY_AFTER_TESTS', + defaultValue: true, + description: 'Tear down infrastructure after a successful run; failed runs are governed by DESTROY_ON_FAILURE' + ) + + // ── Terraform parameters ─────────────────────────────── + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform tfvars for the cluster_nodes module (non-airgap)', + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-0600e22b69546c15c" +instance_type = "t3a.2xlarge" +aws_vpc = "vpc-bfccf4d7" +aws_subnet = "subnet-ee8cac86" +ephemeral_sg_ingress_cidrs = ["${RUNNER_PUBLIC_IP}/32"] +airgap_setup = false +proxy_setup = false +aws_volume_size = 500 +aws_volume_type = "gp3" +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +public_ssh_key = "ssh_public_key.pub" +private_ssh_key = "~/.ssh/${AWS_SSH_PEM_KEY_NAME}" +nodes = [ + { + count = 3 + role = ["etcd", "cp", "worker"] + }, + { + count = 1 + role = ["worker"] + } +]''' + ) + text( + name: 'ANSIBLE_VARIABLES', + description: 'Ansible group_vars for RKE2 install + Rancher Helm deploy (non-airgap). Rancher version keys resolve from the pre-upgrade (FROM) version; rancher_version_upgrade keys hold the resolved target the server is upgraded to via make rancher-upgrade.', + defaultValue: '''--- +kubernetes_version: ${RKE2_VERSION} +cni: calico +kubeconfig_file: "/workspace/${INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml" +cert_manager_version: "1.20.0" +bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +password: "${RANCHER_ADMIN_PASSWORD}" +rancher_version: ${RANCHER_VERSION} +rancher_image_tag: ${RANCHER_IMAGE_TAG} +rancher_chart_repo_url: "${RANCHER_CHART_REPO_URL}" +upgrade_mode: false +rancher_version_upgrade: "${RANCHER_VERSION_UPGRADE}" +rancher_image_tag_upgrade: "${RANCHER_IMAGE_TAG_UPGRADE}" +rancher_chart_repo_upgrade: "${RANCHER_UPGRADE_REPO_NAME}" +k8s_upgrade_mode: false +server_flags: | + ingress-controller: traefik''' + ) + + // ── Test parameters ──────────────────────────────────── + text( + name: 'CATTLE_TEST_CONFIG', + description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', + defaultValue: '''# The downstream per-phase executor can land on a different Jenkins agent +# than this orchestrator; awsNodeTemplate carries the three keys its +# Authorize Security Group stage needs (sg name, vpc, region) so the +# runner's public IP gets authorized on the tofu-created ephemeral SG +# (tf--sg) before the phases talk to the Rancher LB. +awsNodeTemplate: + region: us-east-2 + vpcId: vpc-bfccf4d7 + securityGroup: + - tf-${HOSTNAME_PREFIX}-sg +rancher: + host: ${HOSTNAME_PREFIX}.qa.rancher.space + adminPassword: ${ADMIN_PASSWORD} + insecure: true + # Deliberately false: pre-upgrade resources (Longhorn chart, PVC, workload) + # must survive into the post-upgrade phases; the pipeline environment + # teardown is the cleanup. + cleanup: false + # The qa-infra-automation cluster_nodes RKE2 cluster is Rancher's own + # ("local") cluster; the upgrade suite installs Longhorn on it. + clusterName: local +rancherUpgradeInput: + targetVersion: "${RANCHER_VERSION_UPGRADE}" +# Marker: set only by this pipeline. The suite fails rather than skips when +# this marker is present but the target version is missing (pipeline bug). +rancherUpgradePipeline: + enabled: true''', + ) + booleanParam( + name: 'REPORT_ARTIFACTS', + defaultValue: true, + description: 'Archive token metadata, Qase run config, and failure diagnostics as build artifacts' + ) + + // ── Downstream job parameters ───────────────────────── + string( + name: 'INDIVIDUAL_JOB', + defaultValue: params.INDIVIDUAL_JOB ?: 'go-pit-validation-job', + description: 'Name of the downstream Jenkins job that runs individual Go test suites' + ) + string( + name: 'TEST_PACKAGE', + defaultValue: './validation/rancherupgrade/...', + description: 'Go test package path passed to the downstream job' + ) + string( + name: 'TEST_TIMEOUT', + defaultValue: '60m', + description: 'Timeout for downstream test jobs' + ) + string( + name: 'QASE_REPORTER_SCRIPT', + defaultValue: 'build_qase_reporter_v2.sh', + description: 'Qase reporter script name in pipeline/scripts/' + ) + string( + name: 'QASE_SCHEMA_PREFIX', + defaultValue: 'pit', + description: 'Schema file prefix for Qase operations, for example hostbusters or pit' + ) + } + + environment { + WORKSPACE_NAME = '' + INFRA_CLEANED = 'false' + YQ_VERSION = 'v4.47.2' + // Folder-injected TEST_RUN_NAME (if present) wins; the fallback keeps + // the shared Qase run name readable when it is not injected. + TEST_RUN_NAME = "${env.TEST_RUN_NAME ?: 'Rancher Server Upgrade Validation'}" + } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + + def hostnamePrefix = params.HOSTNAME_PREFIX ?: 'unknown' + currentBuild.displayName = "#${env.BUILD_NUMBER} - ${hostnamePrefix}" + currentBuild.description = "Rancher server upgrade validation for ${hostnamePrefix}" + } + } + } + + stage('Build Docker Images') { + steps { + // rancher-infra-tools — used by make.groovy / infra.* for all make + // targets (library resolves the :latest alias); direct invocations + // in this file use the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${INFRA_TOOLS_IMAGE} -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + // rancher-go-test — used for the qase binary build, token playbook, + // and redaction step, always via the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${GO_TEST_IMAGE} -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." + } + } + + stage('Setup Tools') { + steps { + sh """ + wget -qO ./yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + wget -qO ./yq_checksums https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/checksums + grep -q '^yq_linux_amd64 ' ./yq_checksums + awk '/^yq_linux_amd64 / {print \$19 \" ./yq\"}' ./yq_checksums | sha256sum -c - + rm -f ./yq_checksums + chmod +x ./yq + """ + } + } + + // Auto-detect this runner's public IP so the ephemeral security + // group's SSH/LB ingress rule (ephemeral_sg_ingress_cidrs) allows + // the IP that Ansible/SSH will actually originate from. + stage('Detect Public IP') { + steps { + script { + env.RUNNER_PUBLIC_IP = infrastructure.detectPublicIp() + } + } + } + + // ── Resolve the upgrade window ───────────────────────── + // Empty params resolve at runtime via qa-infra-automation's + // rancher-resolve-version target; explicit pins pass through. + // The chosen lines/regexes and resolved versions are echoed so CI + // logs show exactly which version each stage ran against. + stage('Resolve Rancher Versions') { + steps { + script { + env.RANCHER_FROM_VERSION = params.RANCHER_VERSION_FROM?.trim() ?: '' + env.RANCHER_TO_VERSION = params.RANCHER_VERSION_TO_UPGRADE?.trim() ?: '' + + if (!env.RANCHER_FROM_VERSION) { + echo 'RANCHER_VERSION_FROM empty — resolving latest 2.14.x patch (SOURCE_LINE=2.14, latest final of the line)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: [ + 'SOURCE_LINE=2.14', + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-from.json', text: resolveOut + env.RANCHER_FROM_VERSION = sh( + script: "./yq '.chart_version' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_FROM_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_FROM_IMAGE_TAG = env.RANCHER_FROM_VERSION.startsWith('v') ? env.RANCHER_FROM_VERSION : "v${env.RANCHER_FROM_VERSION}" + echo "RANCHER_VERSION_FROM pinned: ${env.RANCHER_FROM_VERSION} (image tag ${env.RANCHER_FROM_IMAGE_TAG})" + } + + if (!env.RANCHER_TO_VERSION) { + echo 'RANCHER_VERSION_TO_UPGRADE empty — resolving latest released final (no SOURCE_LINE, repo index latest final)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: ["RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}"], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-to.json', text: resolveOut + env.RANCHER_TO_VERSION = sh( + script: "./yq '.chart_version' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_TO_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_TO_IMAGE_TAG = env.RANCHER_TO_VERSION.startsWith('v') ? env.RANCHER_TO_VERSION : "v${env.RANCHER_TO_VERSION}" + echo "RANCHER_VERSION_TO_UPGRADE pinned: ${env.RANCHER_TO_VERSION} (image tag ${env.RANCHER_TO_IMAGE_TAG})" + } + + echo """ + Upgrade window resolved: + FROM (deploy): ${env.RANCHER_FROM_VERSION} (${env.RANCHER_FROM_IMAGE_TAG}) + TO (upgrade): ${env.RANCHER_TO_VERSION} (${env.RANCHER_TO_IMAGE_TAG}) + Chart flavor: ${params.RANCHER_CHART_REPO_FLAVOR} + """.stripIndent() + currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX}" + } + } + } + + stage('Deploy Rancher Pre-Upgrade Environment') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/default" + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def workspaceName = '' + + // ── SSH key setup ────────────────────────── + stage('Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + // Copy public key into the tofu module directory so + // tofu's file() function can read it (restricted to + // the module directory tree). + sh "cp .ssh/*.pub ${tofuModulePath}/ssh_public_key.pub" + } + + // ── Render terraform.tfvars ───────────────── + // Must exist before any make target runs (check-tofu-dir validates it). + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Initialize S3 backend ─────────────────── + stage('Initialize Tofu Backend') { + make.runTarget( + target: 'backend-s3', + dir: env.INFRA_DIR, + makeArgs: [ + "BUCKET=${params.S3_BUCKET_NAME}", + "KEY=${params.S3_KEY_PREFIX}", + "REGION=${params.S3_BUCKET_REGION}" + ] + ) + } + + // ── Create tofu workspace ─────────────────── + stage('Create Workspace') { + workspaceName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_rancher_upgrade_workspace', + suffix: params.HOSTNAME_PREFIX, + includeTimestamp: false + ) + make.runTarget( + target: 'workspace-new', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + env.WORKSPACE_NAME = workspaceName + } + + // ── Provision infra + generate inventory ──── + stage('Provision Infrastructure') { + make.runTarget( + target: 'infra-up', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + } + + // ── Write group_vars/all.yml ──────────────── + // Must exist before make cluster (make infra-up generates + // inventory.yml but not the group_vars subdirectory). + // Rancher version keys carry the resolved FROM version; + // the *_upgrade keys carry the resolved TO version the + // later `make rancher-upgrade` stage targets. + stage('Configure Ansible Variables') { + sh "mkdir -p ${ansiblePath}/inventory/group_vars" + + // Flavor selects the deploy-side repo too: the prime + // leg installs the FROM version from the prime index + // (community stops the 2.14 line at 2.14.3 while prime + // continues it), keeping resolve, deploy, and upgrade + // all on the same artifact line. + def fromChartRepoUrl = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'https://charts.rancher.com/server-charts/prime' : 'https://releases.rancher.com/server-charts/latest' + def upgradeRepoName = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'rancher-prime-upgrade' : 'rancher-upgrade' + + def ansibleVars = infrastructure.parseAndSubstituteVars( + content: params.ANSIBLE_VARIABLES, + envVars: [ + 'RKE2_VERSION': params.RKE2_VERSION, + 'INFRA_DIR': env.INFRA_DIR, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION': env.RANCHER_FROM_VERSION, + 'RANCHER_IMAGE_TAG': env.RANCHER_FROM_IMAGE_TAG, + 'RANCHER_CHART_REPO_URL': fromChartRepoUrl, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION, + 'RANCHER_IMAGE_TAG_UPGRADE': env.RANCHER_TO_IMAGE_TAG, + 'RANCHER_UPGRADE_REPO_NAME': upgradeRepoName + ] + ) + + def rancherVars = infrastructure.parseAndSubstituteVars( + content: '''deploy_rancher: true +install_helm: true +rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" +rancher_bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +rancher_admin_password: "${RANCHER_ADMIN_PASSWORD}" +rancher_use_bundled_system_charts: true +''', + envVars: [ + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD + ] + ) + + // Merge ANSIBLE_VARIABLES + Rancher vars into a single group_vars/all.yml + infrastructure.writeConfig( + path: "${ansiblePath}/inventory/group_vars/all.yml", + content: ansibleVars + '\n' + rancherVars + ) + } + + // ── Deploy RKE2 cluster (ansible; no AWS creds) ── + stage('Deploy RKE2 Cluster') { + make.runTarget( + target: 'cluster', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Deploy pre-upgrade Rancher (helm; no AWS creds) ── + stage('Deploy Rancher') { + make.runTarget( + target: 'rancher', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Capture Rancher hostname ──────────────── + stage('Output Infrastructure Details') { + // Rancher hostname is deterministically constructed from HOSTNAME_PREFIX + // matching the value set in rancherVars (rancher_hostname). + def rancherHostname = "${params.HOSTNAME_PREFIX}.qa.rancher.space" + echo "Rancher Hostname: https://${rancherHostname}" + env.RANCHER_HOSTNAME = rancherHostname + } + } + } + } + } + + stage('Create Test Config and Inject Admin Token') { + steps { + script { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + + // Create cattle-config with variable substitution, including + // the resolved upgrade target for rancherUpgradeInput.targetVersion. + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def resolvedConfig = infrastructure.parseAndSubstituteVars( + content: params.CATTLE_TEST_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION + ] + ) + writeFile file: cattleConfigPath, text: resolvedConfig.trim() + '\n' + } + + // Generate admin token via Ansible playbook + // TODO: replace with 'make token' once that target is added upstream + def containerConfigPath = "/workspace/${cattleConfigPath}" + def tokenTtl = env.RANCHER_TOKEN_TTL ?: '0' + def tokenDescription = "jenkins-rancher-upgrade-${env.BUILD_NUMBER}" + + // Diagnostic logging to validate env + output path before token generation + echo "Token generation context: workspace=${workspace}, config=${containerConfigPath}" + echo "RANCHER_ADMIN_PASSWORD present in params: ${params.RANCHER_ADMIN_PASSWORD ? 'yes' : 'no'}" + + sh """ + docker run --rm --platform linux/amd64 \ + --name generate-token \ + -e RANCHER_ADMIN_PASSWORD \ + -e RANCHER_URL=https://${params.HOSTNAME_PREFIX}.qa.rancher.space \ + -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ + -v ${workspace}:/workspace \ + -w /workspace/${env.INFRA_DIR}/ansible/rke2/default \ + ${GO_TEST_IMAGE} \ + ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ + -e rancher_cattle_config_file=${containerConfigPath} \ + -e rancher_token_ttl=${tokenTtl} \ + -e rancher_token_description=${tokenDescription} \ + -e rancher_token_output_format=json \ + -e rancher_token_output_file=/workspace/rancher-token.json \ + -e rancher_url=https://${params.HOSTNAME_PREFIX}.qa.rancher.space + """ + + def tokenFileExists = fileExists('rancher-token.json') + echo "Token file present after generation: ${tokenFileExists}" + + if (tokenFileExists) { + // Redact secret fields from token JSON via Docker container. + // Pipeline Utility Steps plugin (readJSON/writeJSON) is not installed, + // and python3 is not available on the Jenkins agent host. + sh "docker run --rm -v \$(pwd):/workspace -w /workspace ${GO_TEST_IMAGE} python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'rancher-token-metadata.json' + echo 'Archived redacted token metadata: rancher-token-metadata.json' + } + + // Infrastructure, Rancher deploy, and admin token all succeeded. + // Shared library helpers (infra.*) may have set currentBuild.result to + // UNSTABLE/FAILURE via internal catchError() wrappers for non-fatal warnings, + // which would cause declarative pipeline to skip downstream test stages. + // Reset build result to SUCCESS so the upgrade phases can execute. + if (currentBuild.result && currentBuild.result != 'SUCCESS') { + echo "Resetting currentBuild.result from '${currentBuild.result}' to 'SUCCESS' — token generation succeeded, allowing upgrade phases to run" + currentBuild.result = 'SUCCESS' + } + } else { + error 'rancher-token.json not found after token generation — cannot run test phases without an admin token' + } + } + } + } + + // ── Phase 1: pre-upgrade Longhorn install + workload seed ── + // Starts the shared Qase run once; all three phases report into it. + stage('Pre-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def testRunConfig = "${workspace}/testrunconfig.yaml" + def qaseBinary = "${workspace}/testrun" + + // Build qase testrun binary into the workspace root so the + // agent-side invocation below and testrunconfig.yaml (written + // to the binary's CWD) line up. + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -w /workspace/${env.TESTS_DIR} \ + ${GO_TEST_IMAGE} \ + sh -c 'env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /workspace/testrun ./validation/pipeline/qase/testrun' + """ + + // Start the shared qase test run (completed after the final phase) + sh "${qaseBinary} --startRun=true" + + // Read test run ID + def testRunID = '' + if (fileExists(testRunConfig)) { + testRunID = sh( + script: "./yq '.id' ${testRunConfig}", + returnStdout: true + ).trim() + } + echo "Shared Qase test run started: id=${testRunID}" + if (params.REPORT_ARTIFACTS && fileExists(testRunConfig)) { + archiveArtifacts artifacts: 'testrunconfig.yaml', allowEmptyArchive: true + } + + // NOTE: unlike Jenkinsfile.e2e, rancher.cleanup is NOT set to + // true and no desiredflags are forced here — the upgrade suite's + // cross-phase state must survive, and the pipeline environment + // teardown is the cleanup. + def configYaml = readFile(file: cattleConfigPath) + + echo "Dispatching pre-upgrade phase: GOTEST_TESTCASE ${PRE_UPGRADE_TESTCASE}" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: PRE_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 2: in-place Rancher server upgrade ── + // `make rancher-upgrade` rotates the Rancher admin token and writes the + // new one to ansible/rancher/default-ha/generated.tfvars; the old token + // is revoked, so the test config MUST be re-injected before the + // post-upgrade phases dispatch (verified live: old token 401s). + stage('Upgrade Rancher Server') { + steps { + script { + echo "Upgrading Rancher server: ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (flavor ${params.RANCHER_CHART_REPO_FLAVOR})" + make.runTarget( + target: 'rancher-upgrade', + dir: env.INFRA_DIR, + makeArgs: [ + "ENV=default", + "RANCHER_VERSION_TO_UPGRADE=${env.RANCHER_TO_VERSION}", + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + passAwsCreds: false + ) + + // Re-inject the rotated admin token into the test config + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def generatedTfvars = "${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars" + + if (!fileExists(generatedTfvars)) { + error "Rotated token file not found at ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + def rotatedToken = sh( + script: "awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}", + returnStdout: true + ).trim() + if (!rotatedToken) { + error "Could not extract api_key from ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + echo "Rotated admin token extracted (length ${rotatedToken.length()}); re-injecting into test config" + + // The token never leaves this shell step (no Groovy/env + // interpolation, set +x keeps it out of the xtrace), + // and the injected token is smoke-checked against the + // live API before any phase is dispatched on it. + sh """ + set +x + token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) + # export: yq's strenv() reads the environment, not the + # shell's unexported variables (empty-token 401 root cause). + export token + ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} + status=\$(curl -sk -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) + if [ "\$status" != "200" ]; then + echo "Rotated token rejected by the upgraded Rancher API: HTTP \$status (token length \${#token})" + exit 1 + fi + echo "Rotated token smoke-check: HTTP 200 from the upgraded Rancher API" + """ + echo "Test config now carries the post-upgrade admin token; rancherUpgradeInput.targetVersion=${env.RANCHER_TO_VERSION}" + } + } + } + + // ── Phase 3: post-upgrade assertions ── + stage('Post-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + // Fail fast (without printing the token) if the config + // about to be dispatched does not carry the rotated one. + sh """ + set +x + expected=\$(awk -F'\"' '/^api_key/{print \$2}' ${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars) + actual=\$(./yq '.rancher.adminToken' ${env.TESTS_DIR}/cattle-config.yaml) + if [ "\$expected" != "\$actual" ]; then + echo "Dispatched config adminToken does not match the rotated token (expected length \${#expected}, got length \${#actual})" + exit 1 + fi + echo "Dispatched config carries the rotated admin token (length \${#expected})" + """ + + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${POST_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: POST_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 4: Longhorn chart upgrade through the upgraded catalog ── + // Completes the shared Qase run after this final phase; do not rely on + // the downstream job's own QASE_TEST_RUN_COMPLETE for the shared run. + stage('Longhorn Chart Upgrade Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${CHART_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: CHART_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + + // Complete the shared qase test run + sh "./testrun --startRun=false" + env.QASE_RUN_COMPLETED = 'true' + echo "Shared Qase test run ${testRunID} completed" + } + } + } + } + } + + post { + always { + script { + // Failure diagnostics run first, while the live cluster and + // the docker images still exist: Jenkins evaluates post + // conditions in a fixed order (always before failure, cleanup + // last), so every failure-path action must run in this block, + // ahead of the teardown below. + if (currentBuild.currentResult != 'SUCCESS') { + collectUpgradeDiagnostics() + } + + // Complete the shared Qase run when a phase failed before the + // final stage could close it (no-op once already completed). + if (env.QASE_RUN_COMPLETED != 'true' && fileExists('testrun') && fileExists('testrunconfig.yaml')) { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + sh './testrun --startRun=false' + } + env.QASE_RUN_COMPLETED = 'true' + } + } + + // Single teardown decision point. The failure-path destroy is + // decided here rather than in a failure{} block because always + // runs first: destroyInfrastructure needs workspace_name.txt, + // the tfvars and the infra-tools image, which are only removed + // afterwards in cleanup{}. A failed run is governed by + // DESTROY_ON_FAILURE alone and a successful one by + // DESTROY_AFTER_TESTS, so DESTROY_ON_FAILURE=false on a failed + // run keeps the environment around for debugging even when + // DESTROY_AFTER_TESTS is true. + def buildSucceeded = currentBuild.currentResult == 'SUCCESS' + def shouldDestroy = buildSucceeded ? params.DESTROY_AFTER_TESTS : params.DESTROY_ON_FAILURE + if (shouldDestroy) { + destroyInfrastructure(buildSucceeded ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') + } + + } + } + + cleanup { + script { + // Docker cleanup — cleanup{} runs after always, so the + // teardown above can still resolve the infra-tools image. + sh "docker rmi -f ${GO_TEST_IMAGE} ${INFRA_TOOLS_IMAGE} rancher-go-test:latest rancher-infra-tools:latest || true" + + // Sensitive artifact cleanup — shred or remove credentials from workspace + sh ''' + set +e + + cleanup_file() { + if [ -f "$1" ]; then + if command -v shred >/dev/null 2>&1; then + shred -u "$1" || rm -f "$1" + else + rm -f "$1" + fi + fi + } + + cleanup_dir() { + if [ -d "$1" ]; then + find "$1" -type f -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + rm -rf "$1" + fi + } + + cleanup_file ".go_test_env" + cleanup_file "rancher-token.json" + cleanup_file "rancher-token-metadata.json" + cleanup_file "workspace_name.txt" + cleanup_file "resolve-from.json" + cleanup_file "resolve-to.json" + # cattle-config carries the admin password and the live rotated + # admin token; group_vars/all.yml carries the rancher passwords. + cleanup_file "${TESTS_DIR}/cattle-config.yaml" + + if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then + find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name 'all.yml' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + fi + + cleanup_dir ".ssh" + ''' + } + } + } +} diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md new file mode 100644 index 0000000000..901b0f5b32 --- /dev/null +++ b/validation/rancherupgrade/README.md @@ -0,0 +1,101 @@ +# Rancher Upgrade + +The `rancherupgrade` package validates upgrading the Rancher server from the previous +release line to the latest (currently 2.14.x -> 2.15.x) with interoperability products +installed. The line pair is the pipeline's default, not a suite constraint: the suite +itself is version-agnostic and gates on `rancherUpgradeInput.targetVersion`, while the +pipeline resolves the source via `RANCHER_VERSION_FROM` (empty = latest 2.14.x patch via +`SOURCE_LINE=2.14`) and the target via `RANCHER_VERSION_TO_UPGRADE` (empty = latest +released final). The first vertical is Longhorn: a pre-upgrade +phase installs the Longhorn chart and seeds a checksum workload on a Longhorn volume, +the Rancher server is then upgraded out-of-band (qa-infra-automation `make rancher-upgrade`), +and post-upgrade phases verify the server version, the surviving Longhorn installation, +the workload data, and a Longhorn chart upgrade to the version offered by the upgraded +Rancher's catalog. + +Each phase is a suite test method selected via the `-run` regex and runs as its own +`go test` invocation in its own pipeline stage. All cross-phase state lives in-cluster +under deterministic names (PVC/deployment `rancher-upgrade-longhorn`, data file +`/auto-mnt/checksum-data`), never in process memory. + +## Deliberate deviations + +Do not "fix" the following; both are required by the upgrade workflow (rancher/tests#967): + +- `TearDownSuite` does not call `session.Cleanup()` and the helpers register no cleanup + funcs: pre-phase resources (Longhorn chart, PVC, workload) must survive into the + later phases, which run as separate processes. The pipeline environment teardown is + the cleanup. +- The suite is admin-only: upgrade phases assert server-level state and the pipeline + runs them with the admin token. +- The server upgrade stays out-of-band (a pipeline stage running qa-infra-automation + `make rancher-upgrade`) rather than a step inside the test: the go test phases + assert outcomes only and hold no infra credentials, the upgrade needs the + controller-side qa-infra checkout and its terraform/ansible credentials, and + per-phase Jenkins stage status is the acceptance signal; pulling the upgrade into + the test process would collapse the three stages into one. +- The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy + Rancher once and never upgrade it, so `SetupSuite` skips the suite when + `rancherUpgradeInput.targetVersion` is unset. The dedicated pipeline marks its + config with `rancherUpgradePipeline.enabled: true`; with the marker present but + the target missing, the suite fails (pipeline bug) instead of skipping. + +## Config + +```yaml +rancher: + host: + adminToken: + insecure: true + clusterName: "local" # Rancher's own local cluster: the qa-infra-automation RKE2 HA deploy + +rancherUpgradeInput: + targetVersion: "" +``` +`rancher.clusterName` selects the target cluster. The pipeline uses `local` — Rancher's own +3-node HA RKE2 cluster provisioned by qa-infra-automation — so Longhorn and its volume are +exercised on the server cluster itself, matching what the pipeline deploys. +`rancherUpgradeInput.targetVersion` is required by every phase (the suite skips without it); +the pipeline writes it from the qa-infra-automation resolver output, or set it by hand for +local runs. + +## Running + +Pre-upgrade (installs Longhorn at the 2.14-era latest and seeds the checksum workload; +run against the not-yet-upgraded Rancher): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade" +``` + +Post-upgrade (after `make rancher-upgrade RANCHER_VERSION_TO_UPGRADE= RANCHER_CHART_REPO_FLAVOR=community|prime` +in qa-infra-automation, and with `rancherUpgradeInput.targetVersion` set): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade" +``` + +Chart upgrade: + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade" +``` + +## Notes + +- The target cluster is Rancher's own local cluster (the qa-infra-automation 3-node HA + RKE2 deploy), so the server nodes themselves must be sized for Longhorn plus the later + NeuVector/Observability suites; this is an infra concern and is not configurable here. +- The Rancher server upgrade itself is `make rancher-upgrade` in + [qa-infra-automation](https://github.com/rancher/qa-infra-automation); this package + only runs the pre- and post-upgrade assertions around it. +- `make rancher-upgrade` rotates the admin token and writes the new one to + `ansible/rancher/-ha/generated.tfvars`; the pipeline must copy it into the test + config's `rancher.adminToken` before the post-upgrade phases (the pre-upgrade token + is revoked by the upgrade). +- After a server upgrade the cluster-scoped catalog follower can keep serving the + previous Rancher line's index (e.g. release-v2.14 after upgrading to 2.15) until the + `cattle-cluster-agent` re-syncs. The chart-upgrade phase detects this and fails with + an explicit stale-catalog message when the catalog's latest equals the installed + version, or with "failed to find chart longhorn from the chart repo" on a lookup + miss; restart the cluster agent and re-run the phase. Observed re-sync time ~4 minutes. diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go new file mode 100644 index 0000000000..3b203420d6 --- /dev/null +++ b/validation/rancherupgrade/longhorn_test.go @@ -0,0 +1,182 @@ +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + "github.com/rancher/tests/actions/projects" + "github.com/rancher/tests/actions/storage" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type LonghornUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + cluster *clusters.ClusterMeta +} + +const ( + // checksumWorkloadName is the deterministic name shared by the PVC and the nginx + // deployment created in the pre-upgrade phase and relocated by later phases. + checksumWorkloadName = "rancher-upgrade-longhorn" + + // checksumContent is the shell-safe data written to the checksum file on the + // Longhorn-backed volume before the Rancher upgrade and verified after it. + checksumContent = "rancher-upgrade-longhorn-checksum-v1" +) + +// loadUpgradeInput returns the rancherUpgradeInput section of the test config. +func loadUpgradeInput() *UpgradeInput { + upgradeInput := new(UpgradeInput) + config.LoadConfig(UpgradeInputConfigKey, upgradeInput) + + return upgradeInput +} + +func (s *LonghornUpgradeTestSuite) SetupSuite() { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (Longhorn chart, PVC, workload) must survive into the +// post-upgrade and chart-upgrade phases, which run as separate go test processes. +// The pipeline environment teardown is the cleanup for these resources. +func (s *LonghornUpgradeTestSuite) TearDownSuite() { +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { + logrus.Infof("pre-upgrade phase: creating project on cluster %s", s.cluster.ID) + project, _, err := projects.CreateProjectAndNamespace(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + // Resolve the version from the cluster-scoped catalog and install that: the + // management view can offer versions the cluster-scoped shard does not serve + // yet (rancher-2.15-catalog-sharding-issue.md), and the chart action validates + // against the cluster-scoped index. The pre-phase still yields the 2.14-era + // version by construction, since it runs against the pre-upgrade Rancher. + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + version, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: installing longhorn %s from the %s repo", version, catalog.RancherChartRepo) + + err = charts.InstallLonghornChart(s.client, charts.PayloadOpts{ + Namespace: charts.LonghornNamespace, + Host: s.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: s.cluster, + Version: version, + ProjectID: project.ID, + }, + }, nil) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: longhorn %s installed; seeding checksum workload %s", version, checksumWorkloadName) + + err = storage.CreateChecksumWorkload(s.client, s.cluster.ID, charts.LonghornStorageClass, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: checksum workload %s ready; pre-upgrade state seeded", checksumWorkloadName) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking longhorn survival", cfg.TargetVersion) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.True(s.T(), chart.IsAlreadyInstalled, "longhorn chart must still be installed after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: longhorn chart still installed; verifying workload data") + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: checksum verified on %s", checksumWorkloadName) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + // Resolve the latest version from the cluster-scoped catalog: the management + // view can report versions absent from the cluster-scoped index (see + // rancher-2.15-catalog-sharding-issue.md at the repo root). + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + installed, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.NotEqual(s.T(), + installed.ChartDetails.Spec.Chart.Metadata.Version, latest, + "cluster-scoped catalog still serves the pre-upgrade index: installed longhorn %s equals latest %s; restart the cluster agent and re-run this phase", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + logrus.Infof("chart-upgrade phase: upgrading longhorn %s to %s from the upgraded catalog", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + + installOptions := &charts.InstallOptions{ + Cluster: s.cluster, + Version: latest, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return charts.UpgradeLonghornChart(s.client, installOptions) + }) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + logrus.Infof("chart-upgrade phase: longhorn at %s; verifying workload data", latest) + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: checksum verified on %s", checksumWorkloadName) +} + +func TestLonghornUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(LonghornUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/observability.go b/validation/rancherupgrade/observability.go new file mode 100644 index 0000000000..3cb6b8702a --- /dev/null +++ b/validation/rancherupgrade/observability.go @@ -0,0 +1,541 @@ +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "bytes" + "context" + _ "embed" + "errors" + "fmt" + "io" + "sort" + "strings" + "time" + + provv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1" + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + extencharts "github.com/rancher/shepherd/extensions/charts" + extensionscluster "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/tests/actions/projects" + interoperableCharts "github.com/rancher/tests/interoperability/charts" + "github.com/rancher/tests/interoperability/fleet" + "github.com/rancher/tests/interoperability/observability" + log "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + kwait "k8s.io/apimachinery/pkg/util/wait" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" +) + +const ( + // localClusterName is the Rancher local cluster that always hosts the + // suse-observability server chart, installed into its System project. + localClusterName = "local" + + // systemProjectName is the built-in project hosting the deterministic + // observability namespaces and chart installs on both clusters. + systemProjectName = "System" + + // observabilityServerChartURL is the prime server-charts repository serving + // the suse-observability chart (SUSE Observability is a prime-only product). + observabilityServerChartURL = "https://charts.rancher.com/server-charts/prime/suse-observability" + + // stackstateConfigFileKey is the cattle-config key holding the StackState + // connection settings (url, api key, license, admin password). + stackstateConfigFileKey = "stackstateConfigs" + + // primeImageRegistry is the prime registry the suse-observability server + // images pull from, matching validation/observability. + primeImageRegistry = "registry.rancher.com" + + // observabilitySizingProfile is the chart-native sizing profile for the + // server install; 10-nonha fits the upgrade pipeline's three-node + // cluster. + observabilitySizingProfile = "10-nonha" + + // catalogIndexPollInterval and catalogIndexPollTimeout bound the wait for + // a freshly created ClusterRepo's chart index to be fetched and indexed + // by the catalog controller. + catalogIndexPollInterval = 10 * time.Second + catalogIndexPollTimeout = 5 * time.Minute + + // cattleSystemNamespace hosts the Rancher server pods on the local + // cluster; rancherPodLabel selects them; rancherLogTailLines bounds the + // server log capture on chart action failure, whose bare internal error + // carries no client-visible cause. + cattleSystemNamespace = "cattle-system" + rancherPodLabel = "app=rancher" + // previousLogWindowLines, previousLogTailLines and previousLogErrorLimit + // bound the crash-evidence capture from a restarted container's previous + // instance: a wide window is filtered down to error-marker lines, with a + // short raw tail kept for sequencing context. + rancherLogTailLines = 300 + previousLogWindowLines = 2000 + previousLogTailLines = 15 + previousLogErrorLimit = 40 + eventsDiagnosticsLimit = 40 +) + +const ( + // defaultStorageClassAnnotation marks a StorageClass as the cluster + // default; the suse-observability PVCs request the default class. + defaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class" + + // localPathNamespace and localPathProvisionerName identify the + // local-path-provisioner objects applied by ensureDefaultStorageClass. + localPathNamespace = "local-path-storage" + localPathProvisionerName = "local-path-provisioner" + + // fieldManager is the server-side-apply field manager for manifests + // applied by this suite. + fieldManager = "rancher-tests-rancherupgrade" +) + +// loadStackStateConfig returns the stackstateConfigs section of the test config. +func loadStackStateConfig() *observability.StackStateConfig { + stackstateConfigs := new(observability.StackStateConfig) + config.LoadConfig(stackstateConfigFileKey, stackstateConfigs) + + return stackstateConfigs +} + +// stackstateServerValues builds the suse-observability server chart values +// in the chart's global configuration mode (global.suseObservability), the +// mode the 2.12.0 documentation installs with: license, baseUrl, admin +// password, receiver API key and the native 10-nonha sizing profile, which +// fits the pipeline's three-node cluster (about 7 cores and 27 GiB of +// requests). The legacy stackstate.* sizing path proved unusable there: its +// no-profile fallback sizes the split server components past their own +// memory limits, and a license installed through it never initializes the +// tenant subscription (build 39: every component exiting with +// TenantSubscriptionService$SubscriptionInitializationProhibited). +func stackstateServerValues(stackstateConfigs *observability.StackStateConfig) (map[string]interface{}, error) { + return interoperableCharts.StructToMap(map[string]interface{}{ + "global": map[string]interface{}{ + "imageRegistry": primeImageRegistry, + "suseObservability": map[string]interface{}{ + "license": stackstateConfigs.License, + "baseUrl": stackstateConfigs.Url, + "adminPassword": stackstateConfigs.AdminPassword, + "receiverApiKey": stackstateConfigs.ClusterApiKey, + "sizing": map[string]interface{}{"profile": observabilitySizingProfile}, + }, + }, + }) +} + +// systemProjectID returns the ID of the cluster's System project in the +// cluster-prefixed form the chart actions require: the install action derives +// the target namespace's field.cattle.io/projectId annotation from this value +// (cluster:p-xxx). A bare p-xxx resolves to no project, the namespace's +// project roles never populate, and the action's thirty-second wait for them +// fails; builds 28 through 33 failed exactly there. +func systemProjectID(client *rancher.Client, clusterID string) (string, error) { + systemProject, err := projects.GetProjectByName(client, clusterID, systemProjectName) + if err != nil { + return "", err + } + if systemProject == nil || systemProject.ID == "" { + return "", fmt.Errorf("system project not found on cluster %s", clusterID) + } + + return systemProject.ID, nil +} + +// waitForChartWorkloads waits until every deployment and daemonset in the +// namespace has its expected replica count available, the workload readiness +// gate used by the existing observability chart checks. +func waitForChartWorkloads(client *rancher.Client, clusterID, namespace string) error { + if err := extencharts.WatchAndWaitDeployments(client, clusterID, namespace, meta.ListOptions{}); err != nil { + return err + } + + return extencharts.WatchAndWaitDaemonSets(client, clusterID, namespace, meta.ListOptions{}) +} + +// podStatusClusterName resolves the cluster identifier pods.StatusPods expects, +// mirroring validation/observability: provisioning clusters report the runtime +// cluster name via status, while clusters without a provisioning object (the +// local cluster) fall back to the ID lookup. +func podStatusClusterName(client *rancher.Client, displayName string) (string, error) { + // The lookup error is ignored deliberately: a cluster without a + // provisioning object (local) surfaces as a not-found style error, and the + // fallback below resolves its ID; real API failures surface there too. + clusterObject, _, _ := extensionscluster.GetProvisioningClusterByName(client, displayName, fleet.Namespace) + if clusterObject != nil { + status := new(provv1.ClusterStatus) + if err := steveV1.ConvertToK8sType(clusterObject.Status, status); err != nil { + return "", err + } + + return status.ClusterName, nil + } + + return extensionscluster.GetClusterIDByName(client, displayName) +} + +// latestChartVersionAfterRepoCreate polls the cluster-scoped catalog for the +// latest chart version from a freshly created ClusterRepo. The catalog +// controller fetches and indexes the chart repo's index asynchronously, so the +// first version lookups after a create can fail before the repo is ready. +func latestChartVersionAfterRepoCreate(catalogClient *catalog.Client, chartName, repoName string) (string, error) { + var latestVersion string + err := kwait.PollUntilContextTimeout(context.TODO(), catalogIndexPollInterval, catalogIndexPollTimeout, true, func(context.Context) (bool, error) { + version, err := catalogClient.GetLatestChartVersion(chartName, repoName) + if err != nil { + return false, nil + } + latestVersion = version + return true, nil + }) + if err != nil { + return "", fmt.Errorf("chart %s from repo %s did not appear in the cluster-scoped catalog within %s: %w", chartName, repoName, catalogIndexPollTimeout, err) + } + return latestVersion, nil +} + +// previousStableChartVersion returns the newest stable chart version +// strictly below the repo's latest. The chart-upgrade phase needs a real +// delta: the prime repo's index is identical before and after the Rancher +// server upgrade, so seeding the pre-upgrade phase with the latest version +// leaves the phase nothing to upgrade to (build 43's guard fired exactly +// there). The version list arrives newest-first with -pre entries interleaved. +func previousStableChartVersion(catalogClient *catalog.Client, chartName, repoName string) (string, error) { + versions, err := catalogClient.GetListChartVersions(chartName, repoName) + if err != nil { + return "", err + } + + stable := make([]string, 0, len(versions)) + for _, version := range versions { + if !strings.Contains(version, "-pre") { + stable = append(stable, version) + } + } + if len(stable) == 0 { + return "", fmt.Errorf("chart %s from repo %s has no stable versions", chartName, repoName) + } + if len(stable) < 2 { + return "", fmt.Errorf("chart %s from repo %s has no stable version below %s", chartName, repoName, stable[0]) + } + + return stable[1], nil +} + +// logInstallDiagnostics captures evidence when a chart action fails. The +// action may land on any Rancher server replica, so every replica's log tail +// is captured; the target namespace's conditions (InitialRolesPopulated) and +// the catalog Operation and App resources' statuses carry the helm +// operation's actual error. This is best-effort diagnostic evidence: capture +// failures are logged and swallowed so they never mask the test's own failure. +func logInstallDiagnostics(client *rancher.Client, catalogClient *catalog.Client, clusterID, namespace, appName string) { + k8s, err := clusterK8sClient(client, clusterID) + if err != nil { + log.Warnf("diagnostics: building the local cluster k8s client failed: %v", err) + return + } + + tailLines := int64(rancherLogTailLines) + pods, err := k8s.CoreV1().Pods(cattleSystemNamespace).List(context.TODO(), meta.ListOptions{LabelSelector: rancherPodLabel}) + if err != nil { + log.Warnf("diagnostics: listing pods with label %s in %s failed: %v", rancherPodLabel, cattleSystemNamespace, err) + } else { + for i := range pods.Items { + serverLogs, err := k8s.CoreV1().Pods(cattleSystemNamespace).GetLogs(pods.Items[i].Name, &corev1.PodLogOptions{TailLines: &tailLines}).DoRaw(context.TODO()) + if err != nil { + log.Warnf("diagnostics: fetching rancher logs from pod %s failed: %v", pods.Items[i].Name, err) + continue + } + log.Infof("diagnostics: rancher server log tail from pod %s:\n%s", pods.Items[i].Name, string(serverLogs)) + } + } + + if ns, err := k8s.CoreV1().Namespaces().Get(context.TODO(), namespace, meta.GetOptions{}); err != nil { + log.Warnf("diagnostics: fetching namespace %s failed: %v", namespace, err) + } else { + log.Infof("diagnostics: namespace %s annotations=%v conditions=%v", namespace, ns.Annotations, ns.Status.Conditions) + } + logNamespaceWorkloadState(k8s, namespace) + + if classes, err := k8s.StorageV1().StorageClasses().List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing storage classes failed: %v", err) + } else { + for i := range classes.Items { + log.Infof("diagnostics: storageclass %s default=%v provisioner=%s", classes.Items[i].Name, classes.Items[i].Annotations[defaultStorageClassAnnotation] == "true", classes.Items[i].Provisioner) + } + } + + if ops, err := catalogClient.Operations(namespace).List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing catalog operations in %s failed: %v", namespace, err) + } else { + for i := range ops.Items { + op := ops.Items[i] + log.Infof("diagnostics: operation %s status=%+v", op.Name, op.Status) + // The operation pod's helm container output names the chart-level + // failure directly; capture it when the operation has a pod. + if op.Status.PodName != "" { + helmLogs, err := k8s.CoreV1().Pods(op.Status.PodNamespace).GetLogs(op.Status.PodName, &corev1.PodLogOptions{Container: "helm", TailLines: &tailLines}).DoRaw(context.TODO()) + if err != nil { + log.Warnf("diagnostics: fetching helm logs from operation pod %s failed: %v", op.Status.PodName, err) + } else { + log.Infof("diagnostics: operation %s helm log tail:\n%s", op.Name, string(helmLogs)) + } + } + } + } + + if app, err := catalogClient.Apps(namespace).Get(context.TODO(), appName, meta.GetOptions{}); err != nil { + log.Warnf("diagnostics: fetching app %s in %s failed: %v", appName, namespace, err) + } else { + log.Infof("diagnostics: app %s status=%+v", appName, app.Status) + } +} + +// localPathProvisionerManifest is the upstream local-path-provisioner +// v0.0.30 deploy manifest (rancher/local-path-provisioner) with the default +// StorageClass annotation added; see the resource file for details. +// +//go:embed resources/local-path-provisioner.yaml +var localPathProvisionerManifest []byte + +// clusterRestConfig builds the proxied REST config for the given cluster. +func clusterRestConfig(client *rancher.Client, clusterID string) *rest.Config { + return &rest.Config{ + Host: "https://" + client.RancherConfig.Host + "/k8s/clusters/" + clusterID, + BearerToken: client.RancherConfig.AdminToken, + TLSClientConfig: rest.TLSClientConfig{ + Insecure: client.RancherConfig.Insecure != nil && *client.RancherConfig.Insecure, + CAFile: client.RancherConfig.CAFile, + CAData: []byte(client.RancherConfig.CACerts), + }, + } +} + +// clusterK8sClient builds a Kubernetes client for the given cluster through +// Rancher's proxied API endpoint. +func clusterK8sClient(client *rancher.Client, clusterID string) (*kubernetes.Clientset, error) { + return kubernetes.NewForConfig(clusterRestConfig(client, clusterID)) +} + +// ensureDefaultStorageClass guarantees the cluster has a default +// StorageClass. The upgrade pipeline's RKE2 clusters ship without any +// storage provisioner, so the suse-observability chart's PVCs, which request +// the cluster default class, stay Pending forever and helm --wait burns its +// full deadline (pipeline build 37's diagnostics: every namespace PVC +// Pending, every statefulset blocked on volume binding). When a default +// class already exists (Longhorn-enabled clusters) this is a no-op; +// otherwise the local-path-provisioner manifest is applied and its +// deployment waited on, mirroring how the longhorn module's suite provides +// its own storage. +func ensureDefaultStorageClass(client *rancher.Client, clusterID string) error { + restConfig := clusterRestConfig(client, clusterID) + k8s, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + + classes, err := k8s.StorageV1().StorageClasses().List(context.TODO(), meta.ListOptions{}) + if err != nil { + return err + } + for i := range classes.Items { + if classes.Items[i].Annotations[defaultStorageClassAnnotation] == "true" { + log.Infof("pre-upgrade phase: default storage class %s already present", classes.Items[i].Name) + return nil + } + } + + log.Info("pre-upgrade phase: no default storage class; applying local-path-provisioner") + if err := applyManifest(restConfig, localPathProvisionerManifest); err != nil { + return err + } + + return kwait.PollUntilContextTimeout(context.TODO(), 10*time.Second, 3*time.Minute, true, func(context.Context) (bool, error) { + deployment, err := k8s.AppsV1().Deployments(localPathNamespace).Get(context.TODO(), localPathProvisionerName, meta.GetOptions{}) + if err != nil { + return false, nil //nolint:staticcheck // transient lookup errors are retried by the poll + } + for i := range deployment.Status.Conditions { + if deployment.Status.Conditions[i].Type == appsv1.DeploymentAvailable && deployment.Status.Conditions[i].Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) +} + +// applyManifest server-side applies every document of a multi-object YAML +// manifest to the cluster the REST config points at. +func applyManifest(restConfig *rest.Config, manifest []byte) error { + dyn, err := dynamic.NewForConfig(restConfig) + if err != nil { + return err + } + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return err + } + groupResources, err := restmapper.GetAPIGroupResources(discoveryClient) + if err != nil { + return err + } + mapper := restmapper.NewDiscoveryRESTMapper(groupResources) + + decoder := utilyaml.NewYAMLOrJSONDecoder(bytes.NewReader(manifest), 4096) + for { + object := unstructured.Unstructured{} + if err = decoder.Decode(&object); err != nil { + if errors.Is(err, io.EOF) { + break + } + return err + } + if object.Object == nil { + continue + } + + mapping, err := mapper.RESTMapping(object.GroupVersionKind().GroupKind(), object.GroupVersionKind().Version) + if err != nil { + return fmt.Errorf("mapping %s: %w", object.GroupVersionKind(), err) + } + if _, err = dyn.Resource(mapping.Resource).Namespace(object.GetNamespace()).Apply( + context.TODO(), object.GetName(), &object, meta.ApplyOptions{FieldManager: fieldManager, Force: true}, + ); err != nil { + return fmt.Errorf("applying %s %s: %w", object.GetKind(), object.GetName(), err) + } + } + + return nil +} + +// logNamespaceWorkloadState dumps the readiness of every workload object in +// the chart namespace: pods with their container waiting reasons, restart +// counts, last termination (exit code, reason) and previous-container log +// tails, plus jobs, PVCs, deployments and statefulsets. helm --wait failing +// with "context deadline exceeded" names no culprit; this does. Best-effort +// like logInstallDiagnostics: capture failures are logged and swallowed. +func logNamespaceWorkloadState(k8s *kubernetes.Clientset, namespace string) { + if pods, err := k8s.CoreV1().Pods(namespace).List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing pods in %s failed: %v", namespace, err) + } else { + for i := range pods.Items { + pod := pods.Items[i] + waitStates := make([]string, 0, len(pod.Status.ContainerStatuses)) + crashed := make([]string, 0, len(pod.Status.ContainerStatuses)) + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil { + waitStates = append(waitStates, fmt.Sprintf("%s:%s(%s)", cs.Name, cs.State.Waiting.Reason, cs.State.Waiting.Message)) + } + if cs.RestartCount > 0 { + crashed = append(crashed, cs.Name) + if t := cs.LastTerminationState.Terminated; t != nil { + log.Infof("diagnostics: pod %s container %s last terminated: exitCode=%d reason=%s finished=%s", pod.Name, cs.Name, t.ExitCode, t.Reason, t.FinishedAt.String()) + } + // The previous instance's log names the crash cause (OOM + // stack trace, failed dependency, bad config); JVM boots + // are verbose, so error-marker lines are surfaced from a + // wide window alongside a short raw tail for context. + tail := int64(previousLogWindowLines) + previousLogs, logErr := k8s.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: cs.Name, Previous: true, TailLines: &tail}).DoRaw(context.TODO()) + if logErr != nil { + log.Warnf("diagnostics: fetching previous logs of %s/%s failed: %v", pod.Name, cs.Name, logErr) + } else { + lines := strings.Split(string(previousLogs), "\n") + errorLines := make([]string, 0, previousLogErrorLimit) + for _, line := range lines { + if strings.Contains(line, "ERROR") || strings.Contains(line, "FATAL") || strings.Contains(line, "Exception") || strings.Contains(line, "Caused by") { + errorLines = append(errorLines, line) + } + } + if len(errorLines) > previousLogErrorLimit { + errorLines = errorLines[len(errorLines)-previousLogErrorLimit:] + } + log.Infof("diagnostics: pod %s container %s previous-log errors:\n%s", pod.Name, cs.Name, strings.Join(errorLines, "\n")) + raw := lines + if len(raw) > previousLogTailLines { + raw = raw[len(raw)-previousLogTailLines:] + } + log.Infof("diagnostics: pod %s container %s previous log tail:\n%s", pod.Name, cs.Name, strings.Join(raw, "\n")) + } + } + } + log.Infof("diagnostics: pod %s phase=%s restarts=%v waiting=%v crashedContainers=%v", pod.Name, pod.Status.Phase, containerRestarts(pod.Status.ContainerStatuses), waitStates, crashed) + } + } + + if jobs, err := k8s.BatchV1().Jobs(namespace).List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing jobs in %s failed: %v", namespace, err) + } else { + for i := range jobs.Items { + job := jobs.Items[i] + log.Infof("diagnostics: job %s succeeded=%d failed=%d parallelism=%v conditions=%v", job.Name, job.Status.Succeeded, job.Status.Failed, job.Spec.Parallelism, job.Status.Conditions) + } + } + + if pvcs, err := k8s.CoreV1().PersistentVolumeClaims(namespace).List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing PVCs in %s failed: %v", namespace, err) + } else { + for i := range pvcs.Items { + pvc := pvcs.Items[i] + log.Infof("diagnostics: pvc %s phase=%s", pvc.Name, pvc.Status.Phase) + } + } + + if deps, err := k8s.AppsV1().Deployments(namespace).List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing deployments in %s failed: %v", namespace, err) + } else { + for i := range deps.Items { + dep := deps.Items[i] + log.Infof("diagnostics: deployment %s desired=%d ready=%d available=%d", dep.Name, *dep.Spec.Replicas, dep.Status.ReadyReplicas, dep.Status.AvailableReplicas) + } + } + + if sts, err := k8s.AppsV1().StatefulSets(namespace).List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing statefulsets in %s failed: %v", namespace, err) + } else { + for i := range sts.Items { + st := sts.Items[i] + log.Infof("diagnostics: statefulset %s desired=%d ready=%d current=%d", st.Name, *st.Spec.Replicas, st.Status.ReadyReplicas, st.Status.CurrentReplicas) + } + } + + if events, err := k8s.CoreV1().Events(namespace).List(context.TODO(), meta.ListOptions{}); err != nil { + log.Warnf("diagnostics: listing events in %s failed: %v", namespace, err) + } else { + // Volume-attach stalls and scheduling refusals appear only here: + // the pod sits in ContainerCreating with no container-level evidence + // (build 52's zookeeper stall would have been named by + // FailedAttachVolume). Newest first, capped. + sort.Slice(events.Items, func(i, j int) bool { + return events.Items[i].LastTimestamp.After(events.Items[j].LastTimestamp.Time) + }) + if len(events.Items) > eventsDiagnosticsLimit { + events.Items = events.Items[:eventsDiagnosticsLimit] + } + for i := range events.Items { + e := events.Items[i] + log.Infof("diagnostics: event %s %s/%s %s: %s", e.LastTimestamp.String(), e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message) + } + } +} + +// containerRestarts totals the restart counts of a pod's containers. +func containerRestarts(statuses []corev1.ContainerStatus) int32 { + var restarts int32 + for i := range statuses { + restarts += statuses[i].RestartCount + } + return restarts +} diff --git a/validation/rancherupgrade/observability_upgrade_test.go b/validation/rancherupgrade/observability_upgrade_test.go new file mode 100644 index 0000000000..94cc4548d4 --- /dev/null +++ b/validation/rancherupgrade/observability_upgrade_test.go @@ -0,0 +1,357 @@ +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "context" + "testing" + + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/shepherd/clients/rancher" + extencharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/extensions/workloads/pods" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + interoperableCharts "github.com/rancher/tests/interoperability/charts" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ObservabilityUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + // serverCluster is the Rancher local cluster hosting the suse-observability + // server chart; agentCluster is the shared downstream cluster selected by + // rancher.clusterName (rancher/tests#946 user story 18) hosting the agent. + serverCluster *clusters.ClusterMeta + agentCluster *clusters.ClusterMeta +} + +func (s *ObservabilityUpgradeTestSuite) SetupSuite() { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.serverCluster, err = clusters.NewClusterMeta(client, localClusterName) + require.NoError(s.T(), err) + + s.agentCluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (suse-observability server chart and stackstate agent chart +// with their namespaces) must survive into the post-upgrade and chart-upgrade +// phases, which run as separate go test processes. The pipeline environment +// teardown is the cleanup for these resources. +func (s *ObservabilityUpgradeTestSuite) TearDownSuite() { +} + +func (s *ObservabilityUpgradeTestSuite) TestObservabilityPreUpgrade() { + stackstateConfigs := loadStackStateConfig() + require.NotEmpty(s.T(), stackstateConfigs.Url, "stackstateConfigs.url must be set to install SUSE Observability") + require.NotEmpty(s.T(), stackstateConfigs.ClusterApiKey, "stackstateConfigs.clusterApiKey must be set to install SUSE Observability") + + // Server side, on the Rancher local cluster. Resolve the version from the + // cluster-scoped catalog: the management view can offer versions the + // cluster-scoped shard does not serve yet (rancher-2.15-catalog-sharding-issue.md), + // and the chart action validates against the cluster-scoped index. + serverCatalogClient, err := s.client.GetClusterCatalogClient(s.serverCluster.ID) + require.NoError(s.T(), err) + + _, err = serverCatalogClient.ClusterRepos().Get(context.TODO(), interoperableCharts.StackStateServerChartRepo, meta.GetOptions{}) + if k8sErrors.IsNotFound(err) { + logrus.Infof("pre-upgrade phase: creating %s ClusterRepo at %s", interoperableCharts.StackStateServerChartRepo, observabilityServerChartURL) + err = interoperableCharts.CreateClusterRepo(s.client, serverCatalogClient, interoperableCharts.StackStateServerChartRepo, observabilityServerChartURL) + } + require.NoError(s.T(), err) + + // The ClusterRepo above may have been created seconds ago; its chart index + // is fetched and indexed asynchronously, so poll for index readiness first. + // Then seed the previous stable version, not the latest: the prime repo's + // index does not change across the Rancher server upgrade, so a + // latest-seeded install leaves the chart-upgrade phase without a delta. + _, err = latestChartVersionAfterRepoCreate(serverCatalogClient, interoperableCharts.StackStateServerChartRepo, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + seedServerVersion, err := previousStableChartVersion(serverCatalogClient, interoperableCharts.StackStateServerChartRepo, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + + // Pre-warm the version's chart template: its first fetch downloads the + // chart tarball server-side, and an install action issued before the + // template is cached can outlive the request deadline. + require.NoError(s.T(), interoperableCharts.WaitForChartTemplate(serverCatalogClient, interoperableCharts.StackStateServerChartRepo, interoperableCharts.StackStateServerChartRepo, seedServerVersion)) + + // The install action creates and project-annotates the namespace itself; + // pre-creating it poisons the action's thirty-second watch for the + // InitialRolesPopulated event on an already-quiet namespace. + + serverSystemProjectID, err := systemProjectID(s.client, s.serverCluster.ID) + require.NoError(s.T(), err) + + serverValues, err := stackstateServerValues(stackstateConfigs) + require.NoError(s.T(), err) + + // The chart's PVCs request the cluster default storage class; the + // pipeline's RKE2 clusters ship none, which leaves every PVC Pending and + // helm --wait burning its full deadline. + require.NoError(s.T(), ensureDefaultStorageClass(s.client, s.serverCluster.ID)) + + // Re-entrant install: a partial install from an earlier attempt leaves the + // helm release name taken ("cannot re-use a name that is still in use"). + // A deployed app already satisfies the phase; a stale partial one is + // uninstalled first. + serverInstalled := false + if existing, statusErr := extencharts.GetChartStatus(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace, interoperableCharts.StackStateServerChartRepo); statusErr == nil && existing != nil && existing.ChartDetails != nil { + if existing.ChartDetails.Status.Summary.State == string(catalogv1.StatusDeployed) { + logrus.Infof("pre-upgrade phase: suse-observability server already deployed at %s; skipping install", existing.ChartDetails.Spec.Chart.Metadata.Version) + serverInstalled = true + } else { + logrus.Infof("pre-upgrade phase: stale suse-observability app in state %s; uninstalling before install", existing.ChartDetails.Status.Summary.State) + require.NoError(s.T(), interoperableCharts.UninstallStackStateApp(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace, interoperableCharts.StackStateServerChartRepo)) + } + } + if !serverInstalled { + logrus.Infof("pre-upgrade phase: installing suse-observability server %s in the System project on the local cluster", seedServerVersion) + err = interoperableCharts.InstallStackStateServerChart(s.client, &charts.InstallOptions{ + Cluster: s.serverCluster, + Version: seedServerVersion, + ProjectID: serverSystemProjectID, + }, serverSystemProjectID, serverValues) + if err != nil { + // The install action answers bare internal errors whose cause only + // the Rancher server knows; capture its logs, the namespace state and + // the helm operation's status before failing. + logInstallDiagnostics(s.client, serverCatalogClient, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace, interoperableCharts.StackStateServerChartRepo) + } + require.NoError(s.T(), err) + } + + err = waitForChartWorkloads(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace) + require.NoError(s.T(), err) + + serverPodClusterName, err := podStatusClusterName(s.client, localClusterName) + require.NoError(s.T(), err) + require.Empty(s.T(), pods.StatusPods(s.client, serverPodClusterName), "suse-observability server pods must be healthy after install") + logrus.Infof("pre-upgrade phase: suse-observability server %s healthy; installing the agent", seedServerVersion) + + // Agent side, on the shared downstream cluster selected by rancher.clusterName. + agentCatalogClient, err := s.client.GetClusterCatalogClient(s.agentCluster.ID) + require.NoError(s.T(), err) + + seedAgentVersion, err := previousStableChartVersion(agentCatalogClient, interoperableCharts.StackstateK8sAgent, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + + // As above on the agent side: the install action owns namespace creation. + + agentSystemProjectID, err := systemProjectID(s.client, s.agentCluster.ID) + require.NoError(s.T(), err) + + logrus.Infof("pre-upgrade phase: installing stackstate agent %s on cluster %s", seedAgentVersion, s.agentCluster.ID) + err = interoperableCharts.InstallStackstateAgentChart(s.client, &charts.InstallOptions{ + Cluster: s.agentCluster, + Version: seedAgentVersion, + ProjectID: agentSystemProjectID, + }, stackstateConfigs, agentSystemProjectID) + if err != nil { + // The install action answers bare internal errors whose cause only + // the Rancher server knows; capture the same evidence the server + // install path collects. + logInstallDiagnostics(s.client, agentCatalogClient, s.agentCluster.ID, interoperableCharts.StackstateNamespace, interoperableCharts.StackstateK8sAgent) + } + require.NoError(s.T(), err) + + // Agent reporting is verified the way validation/observability does it: the + // agent's deployments and daemonsets reach their expected replicas and its + // pods report healthy against the server. + err = waitForChartWorkloads(s.client, s.agentCluster.ID, interoperableCharts.StackstateNamespace) + require.NoError(s.T(), err) + + agentPodClusterName, err := podStatusClusterName(s.client, s.client.RancherConfig.ClusterName) + require.NoError(s.T(), err) + require.Empty(s.T(), pods.StatusPods(s.client, agentPodClusterName), "stackstate agent pods must be healthy and reporting after install") + logrus.Infof("pre-upgrade phase: stackstate agent %s reporting to %s; pre-upgrade state seeded", seedAgentVersion, stackstateConfigs.Url) +} + +func (s *ObservabilityUpgradeTestSuite) TestObservabilityPostUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking suse-observability server health", cfg.TargetVersion) + + serverChart, err := extencharts.GetChartStatus(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + require.True(s.T(), serverChart.IsAlreadyInstalled, "suse-observability server chart must still be installed after the Rancher server upgrade") + + err = waitForChartWorkloads(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace) + require.NoError(s.T(), err) + + serverPodClusterName, err := podStatusClusterName(s.client, localClusterName) + require.NoError(s.T(), err) + require.Empty(s.T(), pods.StatusPods(s.client, serverPodClusterName), "suse-observability server pods must be healthy after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: suse-observability server healthy; checking agent reporting continuity") + + agentChart, err := extencharts.GetChartStatus(s.client, s.agentCluster.ID, interoperableCharts.StackstateNamespace, interoperableCharts.StackstateK8sAgent) + require.NoError(s.T(), err) + require.True(s.T(), agentChart.IsAlreadyInstalled, "stackstate agent chart must still be installed after the Rancher server upgrade") + + err = waitForChartWorkloads(s.client, s.agentCluster.ID, interoperableCharts.StackstateNamespace) + require.NoError(s.T(), err) + + agentPodClusterName, err := podStatusClusterName(s.client, s.client.RancherConfig.ClusterName) + require.NoError(s.T(), err) + require.Empty(s.T(), pods.StatusPods(s.client, agentPodClusterName), "stackstate agent pods must still be healthy and reporting after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: monitoring continuity confirmed; stackstate agent still reporting to the server") +} + +func (s *ObservabilityUpgradeTestSuite) TestObservabilityChartUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + stackstateConfigs := loadStackStateConfig() + require.NotEmpty(s.T(), stackstateConfigs.Url, "stackstateConfigs.url must be set to upgrade SUSE Observability") + require.NotEmpty(s.T(), stackstateConfigs.ClusterApiKey, "stackstateConfigs.clusterApiKey must be set to upgrade SUSE Observability") + + // Resolve the latest versions from the cluster-scoped catalogs, not the + // management view: the management view can report versions absent from the + // cluster-scoped index (rancher-2.15-catalog-sharding-issue.md). The server + // resolves via the prime server-charts ClusterRepo on the local cluster, the + // agent via the suse-observability repo on the downstream cluster. + serverCatalogClient, err := s.client.GetClusterCatalogClient(s.serverCluster.ID) + require.NoError(s.T(), err) + latestServerVersion, err := serverCatalogClient.GetLatestChartVersion(interoperableCharts.StackStateServerChartRepo, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + + agentCatalogClient, err := s.client.GetClusterCatalogClient(s.agentCluster.ID) + require.NoError(s.T(), err) + latestAgentVersion, err := agentCatalogClient.GetLatestChartVersion(interoperableCharts.StackstateK8sAgent, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + + // Pre-warm the newly offered versions' templates: a version's first + // template fetch downloads the chart tarball server-side, and an upgrade + // action issued before the template is cached can outlive the request + // deadline. + require.NoError(s.T(), interoperableCharts.WaitForChartTemplate(serverCatalogClient, interoperableCharts.StackStateServerChartRepo, interoperableCharts.StackStateServerChartRepo, latestServerVersion)) + require.NoError(s.T(), interoperableCharts.WaitForChartTemplate(agentCatalogClient, interoperableCharts.StackStateServerChartRepo, interoperableCharts.StackstateK8sAgent, latestAgentVersion)) + + installedServer, err := extencharts.GetChartStatus(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + + installedAgent, err := extencharts.GetChartStatus(s.client, s.agentCluster.ID, interoperableCharts.StackstateNamespace, interoperableCharts.StackstateK8sAgent) + require.NoError(s.T(), err) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + require.NotEqual(s.T(), + installedServer.ChartDetails.Spec.Chart.Metadata.Version, latestServerVersion, + "local cluster-scoped catalog still serves the pre-upgrade index: installed suse-observability %s equals latest %s; restart the local cluster agent and re-run this phase", + installedServer.ChartDetails.Spec.Chart.Metadata.Version, latestServerVersion) + require.NotEqual(s.T(), + installedAgent.ChartDetails.Spec.Chart.Metadata.Version, latestAgentVersion, + "downstream cluster-scoped catalog still serves the pre-upgrade index: installed stackstate agent %s equals latest %s; restart the downstream cluster agent and re-run this phase", + installedAgent.ChartDetails.Spec.Chart.Metadata.Version, latestAgentVersion) + + // Re-apply the install-time server values rather than a blind version bump: + // upgrading on chart defaults would drop the sizing profile and the + // stackstate service credentials (rancher/tests#946 Further Notes on + // per-product values migration). + serverValues, err := stackstateServerValues(stackstateConfigs) + require.NoError(s.T(), err) + + serverSystemProjectID, err := systemProjectID(s.client, s.serverCluster.ID) + require.NoError(s.T(), err) + agentSystemProjectID, err := systemProjectID(s.client, s.agentCluster.ID) + require.NoError(s.T(), err) + + // Upgrade the server first so the agent upgrades against a live, current server. + logrus.Infof("chart-upgrade phase: upgrading suse-observability server %s to %s", + installedServer.ChartDetails.Spec.Chart.Metadata.Version, latestServerVersion) + serverInstallOptions := &charts.InstallOptions{ + Cluster: s.serverCluster, + Version: latestServerVersion, + ProjectID: serverSystemProjectID, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return interoperableCharts.UpgradeStackStateServerChart(s.client, serverInstallOptions, serverValues) + }) + if err != nil { + // Upgrade failures surface the same bare internal errors as installs; + // capture the helm operation, app state, workload state and rancher + // tails before failing (build 49's all-module run timed out here + // with no in-band evidence). + logInstallDiagnostics(s.client, serverCatalogClient, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace, interoperableCharts.StackStateServerChartRepo) + } + require.NoError(s.T(), err) + + upgradedServer, err := extencharts.GetChartStatus(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace, interoperableCharts.StackStateServerChartRepo) + require.NoError(s.T(), err) + require.Equal(s.T(), latestServerVersion, upgradedServer.ChartDetails.Spec.Chart.Metadata.Version) + + err = waitForChartWorkloads(s.client, s.serverCluster.ID, interoperableCharts.StackStateServerNamespace) + require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: suse-observability server at %s and healthy; upgrading the agent", latestServerVersion) + + logrus.Infof("chart-upgrade phase: upgrading stackstate agent %s to %s", + installedAgent.ChartDetails.Spec.Chart.Metadata.Version, latestAgentVersion) + agentInstallOptions := &charts.InstallOptions{ + Cluster: s.agentCluster, + Version: latestAgentVersion, + ProjectID: agentSystemProjectID, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return interoperableCharts.UpgradeStackstateAgentChart(s.client, agentInstallOptions, stackstateConfigs, agentSystemProjectID) + }) + if err != nil { + // Same evidence capture as the server upgrade path above. + logInstallDiagnostics(s.client, agentCatalogClient, s.agentCluster.ID, interoperableCharts.StackstateNamespace, interoperableCharts.StackstateK8sAgent) + } + require.NoError(s.T(), err) + + upgradedAgent, err := extencharts.GetChartStatus(s.client, s.agentCluster.ID, interoperableCharts.StackstateNamespace, interoperableCharts.StackstateK8sAgent) + require.NoError(s.T(), err) + require.Equal(s.T(), latestAgentVersion, upgradedAgent.ChartDetails.Spec.Chart.Metadata.Version) + + err = waitForChartWorkloads(s.client, s.agentCluster.ID, interoperableCharts.StackstateNamespace) + require.NoError(s.T(), err) + + agentPodClusterName, err := podStatusClusterName(s.client, s.client.RancherConfig.ClusterName) + require.NoError(s.T(), err) + require.Empty(s.T(), pods.StatusPods(s.client, agentPodClusterName), "stackstate agent pods must be healthy and reporting after the chart upgrade") + logrus.Infof("chart-upgrade phase: stackstate agent at %s and reporting; observability chart upgrade complete", latestAgentVersion) +} + +func TestObservabilityUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(ObservabilityUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/rancherupgrade.go b/validation/rancherupgrade/rancherupgrade.go new file mode 100644 index 0000000000..5d3aeb112a --- /dev/null +++ b/validation/rancherupgrade/rancherupgrade.go @@ -0,0 +1,29 @@ +// Package rancherupgrade validates the Rancher server upgrade path (2.14.x -> latest) +// with interoperability products installed. Each phase (pre-upgrade, post-upgrade, +// chart-upgrade) is a suite test method selected via the -run regex and runs as its +// own go test invocation, so all cross-phase state lives in-cluster under the +// deterministic names defined here. +package rancherupgrade + +const ( + // UpgradeInputConfigKey is the cattle-config key holding upgrade-phase inputs. + UpgradeInputConfigKey = "rancherUpgradeInput" + + // PipelineConfigKey marks a config as authored by the rancher-upgrade + // pipeline; generic PIT legs never set it. + PipelineConfigKey = "rancherUpgradePipeline" +) + +// UpgradeInput holds per-run upgrade parameters. TargetVersion is the Rancher +// version the server is upgraded to; written into the config by the pipeline +// (qa-infra-automation resolver output) or by hand for local runs. +type UpgradeInput struct { + TargetVersion string `json:"targetVersion" yaml:"targetVersion"` +} + +// PipelineInput is the marker section the rancher-upgrade pipeline writes into +// its cattle-config; the suite fails rather than skips when it is present but +// the upgrade target is missing (pipeline bug, not a generic PIT leg). +type PipelineInput struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} diff --git a/validation/rancherupgrade/resources/local-path-provisioner.yaml b/validation/rancherupgrade/resources/local-path-provisioner.yaml new file mode 100644 index 0000000000..e5f59a3077 --- /dev/null +++ b/validation/rancherupgrade/resources/local-path-provisioner.yaml @@ -0,0 +1,165 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: local-path-provisioner-role + namespace: local-path-storage +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role +rules: + - apiGroups: [""] + resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: local-path-provisioner-bind + namespace: local-path-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: rancher/local-path-provisioner:v0.0.30 + imagePullPolicy: IfNotPresent + command: + - local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONFIG_MOUNT_PATH + value: /etc/config/ + volumes: + - name: config-volume + configMap: + name: local-path-config + +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-path + annotations: + # Added so the chart's PVCs, which request the cluster default class, + # bind; the upstream manifest leaves this to the platform (k3s) to set. + storageclass.kubernetes.io/is-default-class: "true" +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete + +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/opt/local-path-provisioner"] + } + ] + } + setup: |- + #!/bin/sh + set -eu + mkdir -m 0777 -p "$VOL_DIR" + teardown: |- + #!/bin/sh + set -eu + rm -rf "$VOL_DIR" + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + priorityClassName: system-node-critical + tolerations: + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + containers: + - name: helper-pod + image: busybox + imagePullPolicy: IfNotPresent diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml new file mode 100644 index 0000000000..6e6231f92c --- /dev/null +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -0,0 +1,135 @@ +- projects: [RANCHERINT] + suite: Rancher Upgrade/Longhorn + cases: + - title: "Longhorn pre-upgrade setup" + description: "Install Longhorn and seed a checksum workload on a Longhorn volume before the Rancher server upgrade" + automation: 2 + steps: + - action: "Create a project on the Rancher local cluster" + data: "" + expectedresult: "The project is created successfully" + position: 1 + - action: "Install the Longhorn chart at the latest version offered by the pre-upgrade Rancher" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "Longhorn installs and its deployments and daemonsets become Ready" + position: 2 + - action: "Create the checksum workload on a Longhorn-backed PVC" + data: "1GiB PVC and nginx deployment named rancher-upgrade-longhorn with data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound, deployment rolls out, and the checksum data is written to the Longhorn volume" + position: 3 + custom_field: + "15": "TestLonghornPreUpgrade" + - title: "Longhorn post-upgrade validation" + description: "Verify Longhorn and its volume data survive the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the Longhorn chart installation state" + data: "Chart: longhorn in longhorn-system" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the checksum workload on the Longhorn volume" + data: "PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound and the sha256 of the data file matches the pre-upgrade content" + position: 3 + custom_field: + "15": "TestLonghornPostUpgrade" + - title: "Longhorn chart upgrade after Rancher upgrade" + description: "Upgrade Longhorn to the version offered by the upgraded Rancher catalog and verify workload data" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest Longhorn chart version from the cluster-scoped catalog" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "The latest cluster-scoped chart version is resolved and differs from the installed pre-upgrade version" + position: 2 + - action: "Upgrade the Longhorn chart with retry" + data: "longhorn and longhorn-crd to the latest version" + expectedresult: "The longhorn chart reaches Deployed state and workloads become Ready" + position: 3 + - action: "Verify the chart version and the checksum workload" + data: "Chart version equals latest; PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "Installed chart version matches latest and the sha256 of the data file still matches the pre-upgrade content" + position: 4 + custom_field: + "15": "TestLonghornChartUpgrade" + +- projects: [RANCHERINT] + suite: Rancher Upgrade/Observability + cases: + - title: "SUSE Observability pre-upgrade setup" + description: "Install the SUSE Observability server and a stackstate agent reporting to it before the Rancher server upgrade" + automation: 2 + steps: + - action: "Ensure the prime server-charts ClusterRepo exists on the local cluster" + data: "ClusterRepo suse-observability at https://charts.rancher.com/server-charts/prime/suse-observability" + expectedresult: "The ClusterRepo is available to the cluster-scoped catalog" + position: 1 + - action: "Install the suse-observability server chart in the System project on the local cluster" + data: "Chart: suse-observability at the previous stable cluster-scoped version (one below latest, leaving a delta for the chart-upgrade phase) with global base values and the 10-nonha sizing profile" + expectedresult: "The chart reaches Deployed state, its deployments and daemonsets become Ready, and its pods are healthy" + position: 2 + - action: "Install the stackstate agent chart on the shared downstream cluster" + data: "Chart: suse-observability-agent from the suse-observability repo, at the previous stable version, with cluster, apiKey and url values from stackstateConfigs" + expectedresult: "The chart reaches Deployed state, its deployments and daemonsets become Ready, and its pods are healthy" + position: 3 + - action: "Verify the agent is reporting to the server" + data: "Agent workloads ready in the stackstate namespace on the downstream cluster" + expectedresult: "The agent pods stay healthy against the server URL, seeding pre-upgrade monitoring state" + position: 4 + custom_field: + "15": "TestObservabilityPreUpgrade" + - title: "SUSE Observability post-upgrade validation" + description: "Verify the SUSE Observability server stays healthy and agents keep reporting across the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the suse-observability server chart installation state" + data: "Chart: suse-observability in the suse-observability namespace on the local cluster" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the server workloads are healthy" + data: "Deployments and daemonsets in the suse-observability namespace on the local cluster" + expectedresult: "All deployments and daemonsets reach their expected replicas and the server pods are healthy" + position: 3 + - action: "Verify monitoring continuity via the agent" + data: "Chart: suse-observability-agent in the stackstate namespace on the downstream cluster" + expectedresult: "The agent chart is still installed, its workloads are Ready, and its pods are healthy and reporting" + position: 4 + custom_field: + "15": "TestObservabilityPostUpgrade" + - title: "SUSE Observability chart upgrade after Rancher upgrade" + description: "Upgrade the suse-observability server and stackstate agent charts to the versions offered by the upgraded Rancher catalog and re-verify health" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest server and agent chart versions from the cluster-scoped catalogs" + data: "suse-observability from the prime server-charts ClusterRepo on the local cluster; suse-observability-agent from the suse-observability repo on the downstream cluster" + expectedresult: "The latest cluster-scoped versions are resolved and differ from the installed pre-upgrade versions" + position: 2 + - action: "Upgrade the suse-observability server chart with the install-time values" + data: "Global base values and the 10-nonha sizing profile re-applied to avoid dropping the configuration mode and service credentials" + expectedresult: "The server chart reaches Deployed state at the latest version and its workloads become Ready" + position: 3 + - action: "Upgrade the stackstate agent chart" + data: "suse-observability-agent upgraded to the latest version with cluster, apiKey and url values" + expectedresult: "The agent chart reaches Deployed state at the latest version and its workloads become Ready" + position: 4 + - action: "Verify post-upgrade health and reporting" + data: "Server and agent chart versions equal the resolved latest versions" + expectedresult: "Both charts report the latest versions and the agent pods are healthy and reporting to the server" + position: 5 + custom_field: + "15": "TestObservabilityChartUpgrade"