diff --git a/internal/app/app.go b/internal/app/app.go index 66c499b7..63ee16c0 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -39,6 +39,7 @@ type App struct { sysLog *syslog.Writer config *config.Config dcs dcs.DCS + appDCS IAppDCS cluster *mysql.Cluster filelock *flock.Flock t *Timings @@ -148,6 +149,7 @@ func (app *App) connectDCS() error { if err != nil { return fmt.Errorf("failed to connect to zkDCS: %w", err) } + app.appDCS = NewAppDCS(app.dcs, app.config, app.logger) return nil } @@ -212,7 +214,7 @@ func (app *App) healthChecker(ctx context.Context) { hc := app.getLocalNodeState() logFile, maxLogPos = hc.UpdateBinlogStatus(logFile, maxLogPos) app.logger.Info().Msgf("healthcheck: %v", hc) - err := app.dcs.SetEphemeral(dcs.JoinPath(pathHealthPrefix, app.config.Hostname), hc) + err := app.SetHealthState(app.config.Hostname, hc) if err != nil { app.logger.Error().Err(err).Msg("healthcheck: failed to set status to dcs") } @@ -609,7 +611,7 @@ func (app *App) stateManager() appState { // Signal to dcs that we have set light maintenance mode if !maintenance.MaintAcquired() { maintenance.MySyncPaused = true - err := app.dcs.Set(pathMaintenance, maintenance) + err := app.SetMaintenance(maintenance) if err != nil { app.logger.Error().Err(err).Msg("failed to enter light maintenance mode") @@ -634,7 +636,7 @@ func (app *App) stateManager() appState { // check if switchover required or in progress switchover := new(Switchover) - if err := app.dcs.Get(pathCurrentSwitch, switchover); err == nil { + if err := app.GetCurrentSwitchover(switchover); err == nil { // failover via DCS is suppressed during light maintenance (only manual switchover is allowed) if lightMaintenance && switchover.MasterTransition == FailoverTransition { app.logger.Info().Msgf("failover suppressed by light maintenance mode") @@ -664,7 +666,7 @@ func (app *App) stateManager() appState { return stateManager } err = app.performSwitchover(clusterState, activeNodes, switchover, master) - if errors.Is(app.dcs.Get(pathCurrentSwitch, new(Switchover)), dcs.ErrNotFound) { + if errors.Is(app.GetCurrentSwitchover(new(Switchover)), dcs.ErrNotFound) { app.logger.Error().Msgf("switchover was aborted") } else { if err != nil { @@ -920,15 +922,8 @@ func (app *App) approveFailover(clusterState, clusterStateDcs map[string]*nodest return err } - var lastSwitchover Switchover - err = app.dcs.Get(pathLastSwitch, &lastSwitchover) - if !errors.Is(err, dcs.ErrNotFound) { - if err != nil { - return err - } - if lastSwitchover.Result == nil { - return fmt.Errorf("another switchover in progress. this should never happen") - } + lastSwitchover := app.appDCS.GetLastSwitchover() + if lastSwitchover.Result != nil { timeAfterLastSwitchover := time.Since(lastSwitchover.Result.FinishedAt) if timeAfterLastSwitchover < app.config.FailoverCooldown && lastSwitchover.Cause == CauseAuto { return fmt.Errorf("not enough time from last failover %s (cooldown %s)", lastSwitchover.Result.FinishedAt, app.config.FailoverCooldown) @@ -1151,7 +1146,7 @@ func (app *App) updateActiveNodes(clusterState, clusterStateDcs map[string]*node app.disableSemiSyncIfNonNeeded(node, state) } // then update DCS - err = app.dcs.Set(pathActiveNodes, activeNodes) + err = app.SetActiveNodes(activeNodes) if err != nil { app.logger.Error().Err(err).Msg("update active nodes: failed to update active nodes in dcs") return err @@ -1226,7 +1221,7 @@ func (app *App) updateActiveNodes(clusterState, clusterStateDcs map[string]*node } // then update DCS - err = app.dcs.Set(pathActiveNodes, activeNodes) + err = app.SetActiveNodes(activeNodes) if err != nil { app.logger.Error().Err(err).Msg("update active nodes: failed to update active nodes in dcs") return err @@ -1647,7 +1642,7 @@ func (app *App) performSwitchover(clusterState map[string]*nodestate.NodeState, } // set new master in dcs - err = app.dcs.Set(pathMasterNode, newMaster) + _, err = app.SetMasterHost(newMaster) if err != nil || app.emulateError("promote_set_to_dcs") { return fmt.Errorf("failed to set new master to dcs: %w", err) } @@ -2213,13 +2208,13 @@ func (app *App) enterMaintenance(maintenance *Maintenance, master string) error if err != nil { return err } - err = app.dcs.Delete(pathActiveNodes) + err = app.DeleteActiveNodes() if err != nil { return err } } maintenance.MySyncPaused = true - return app.dcs.Set(pathMaintenance, maintenance) + return app.SetMaintenance(maintenance) } func (app *App) leaveMaintenance() error { @@ -2252,7 +2247,7 @@ func (app *App) leaveMaintenance() error { if len(activeNodes) == 0 { return ErrNoActiveNodes } - return app.dcs.Delete(pathMaintenance) + return app.DeleteMaintenance() } func (app *App) performChangeMaster(host, master string) error { @@ -2459,7 +2454,7 @@ func (app *App) getClusterStateFromDcs() (map[string]*nodestate.NodeState, error hosts := app.cluster.AllNodeHosts() getter := func(host string) (*nodestate.NodeState, error) { nodeState := new(nodestate.NodeState) - err := app.dcs.Get(dcs.JoinPath(pathHealthPrefix, host), nodeState) + err := app.GetHealthState(host, nodeState) if err != nil && !errors.Is(err, dcs.ErrNotFound) { return nil, err } @@ -2481,7 +2476,7 @@ func (app *App) waitForCatchUp(node *mysql.Node, gtidset gtids.GTIDSet, timeout return true, nil } switchover := new(Switchover) - if errors.Is(app.dcs.Get(pathCurrentSwitch, switchover), dcs.ErrNotFound) { + if errors.Is(app.GetCurrentSwitchover(switchover), dcs.ErrNotFound) { return false, nil } if app.CheckAsyncSwitchAllowed(node, switchover) { @@ -2527,24 +2522,11 @@ func (app *App) stopReplicationOnMaster(masterNode *mysql.Node) error { } func (app *App) fetchCascadeNodeConfigurations() (map[string]mysql.CascadeNodeConfiguration, error) { - cascadeTopology := make(map[string]mysql.CascadeNodeConfiguration) - - hosts, err := app.dcs.GetChildren(dcs.PathCascadeNodesPrefix) + cascadeTopology, err := app.FetchCascadeNodeConfigurations() if err != nil { app.logger.Warn().Msgf("repair: Failed to fetch CascadeNodeConfigurations") - return cascadeTopology, err } - for _, host := range hosts { - var cnc mysql.CascadeNodeConfiguration - err := app.dcs.Get(dcs.JoinPath(dcs.PathCascadeNodesPrefix, host), &cnc) - if err != nil { - app.logger.Warn().Msgf("repair: Failed to fetch CascadeNodeConfiguration for %s", host) - return cascadeTopology, err - } - cascadeTopology[host] = cnc - } - - return cascadeTopology, nil + return cascadeTopology, err } func (app *App) getNodePositions(activeNodes []string) ([]nodePosition, error) { @@ -2585,8 +2567,7 @@ func (app *App) getNodePositions(activeNodes []string) ([]nodePosition, error) { } } - var nc mysql.NodeConfiguration - err = app.dcs.Get(dcs.JoinPath(pathHANodes, host), &nc) + nc, err := app.GetNodeConfiguration(host) if err != nil { if !errors.Is(err, dcs.ErrNotFound) && !errors.Is(err, dcs.ErrMalformed) { return fmt.Errorf("failed to get priority for host %s: %w", host, err) diff --git a/internal/app/app_dcs.go b/internal/app/app_dcs.go index a32f769d..06e81b9a 100644 --- a/internal/app/app_dcs.go +++ b/internal/app/app_dcs.go @@ -1,135 +1,113 @@ package app import ( - "errors" - "fmt" "time" - "github.com/yandex/mysync/internal/dcs" + nodestate "github.com/yandex/mysync/internal/app/node_state" "github.com/yandex/mysync/internal/mysql" - "github.com/yandex/mysync/internal/util" ) -// GetActiveNodes returns master + alive running replicas +// The methods below are thin wrappers on *App that delegate to app.appDCS. +// They exist so that existing callers (cli_*.go, recovery.go, async.go, etc.) +// continue to work without modification. New code should call app.appDCS directly. + +// GetActiveNodes returns master + alive running replicas. func (app *App) GetActiveNodes() ([]string, error) { - var activeNodes []string - err := app.dcs.Get(pathActiveNodes, &activeNodes) - if err != nil { - if errors.Is(err, dcs.ErrNotFound) || errors.Is(err, dcs.ErrMalformed) { - return nil, nil - } - return nil, fmt.Errorf("failed to get active nodes from zk %w", err) - } - return activeNodes, nil + return app.appDCS.GetActiveNodes() } -func (app *App) GetClusterCascadeFqdnsFromDcs() ([]string, error) { - fqdns, err := app.dcs.GetChildren(dcs.PathCascadeNodesPrefix) - if errors.Is(err, dcs.ErrNotFound) { - return make([]string, 0), nil - } - if err != nil { - return nil, err - } +// SetActiveNodes writes the active nodes list to ZK. +func (app *App) SetActiveNodes(nodes []string) error { + return app.appDCS.SetActiveNodes(nodes) +} - return fqdns, nil +// DeleteActiveNodes removes the active nodes list from ZK. +func (app *App) DeleteActiveNodes() error { + return app.appDCS.DeleteActiveNodes() } +// SetHealthState writes the ephemeral per-host health state to ZK. +func (app *App) SetHealthState(host string, state *nodestate.NodeState) error { + return app.appDCS.SetHealthState(host, state) +} + +// GetHealthState reads the per-host health state from ZK. +func (app *App) GetHealthState(host string, state *nodestate.NodeState) error { + return app.appDCS.GetHealthState(host, state) +} + +// SetMaintenance writes the maintenance record to ZK. +func (app *App) SetMaintenance(maintenance *Maintenance) error { + return app.appDCS.SetMaintenance(maintenance) +} + +// DeleteMaintenance removes the maintenance record from ZK. +func (app *App) DeleteMaintenance() error { + return app.appDCS.DeleteMaintenance() +} + +// FetchCascadeNodeConfigurations reads all cascade node configurations from ZK. +func (app *App) FetchCascadeNodeConfigurations() (map[string]mysql.CascadeNodeConfiguration, error) { + return app.appDCS.FetchCascadeNodeConfigurations() +} + +// GetNodeConfiguration reads the HA node configuration (priority etc.) from ZK. +func (app *App) GetNodeConfiguration(host string) (mysql.NodeConfiguration, error) { + return app.appDCS.GetNodeConfiguration(host) +} + +// GetClusterCascadeFqdnsFromDcs returns cascade node FQDNs stored in ZK. +func (app *App) GetClusterCascadeFqdnsFromDcs() ([]string, error) { + return app.appDCS.GetClusterCascadeFqdnsFromDcs() +} + +// GetMaintenance returns the current maintenance record from ZK. func (app *App) GetMaintenance() (*Maintenance, error) { - maintenance := new(Maintenance) - err := app.dcs.Get(pathMaintenance, maintenance) - if err != nil { - return nil, err - } - return maintenance, err + return app.appDCS.GetMaintenance() } +// GetHostsOnRecovery returns hosts currently marked for recovery. func (app *App) GetHostsOnRecovery() ([]string, error) { - hosts, err := app.dcs.GetChildren(pathRecovery) - if errors.Is(err, dcs.ErrNotFound) { - return nil, nil - } - return hosts, err + return app.appDCS.GetHostsOnRecovery() } +// ClearRecovery removes the recovery marker for a host. func (app *App) ClearRecovery(host string) error { - return app.dcs.Delete(dcs.JoinPath(pathRecovery, host)) + return app.appDCS.ClearRecovery(host) } +// SetRecovery marks a host for recovery and removes it from active nodes. func (app *App) SetRecovery(host string) error { - activeNodes, err := app.GetActiveNodes() - if err != nil { - return err - } - activeNodes = util.FilterStrings(activeNodes, func(n string) bool { - return n != host - }) - err = app.dcs.Set(pathActiveNodes, activeNodes) - if err != nil { - return err - } - err = app.dcs.Create(pathRecovery, nil) - if err != nil && !errors.Is(err, dcs.ErrExists) { - return err - } - err = app.dcs.Create(dcs.JoinPath(pathRecovery, host), nil) - if err != nil && !errors.Is(err, dcs.ErrExists) { - return err - } - return nil + return app.appDCS.SetRecovery(host) } +// IsRecoveryNeeded returns true if the host has a recovery marker in ZK. func (app *App) IsRecoveryNeeded(host string) bool { - err := app.dcs.Get(dcs.JoinPath(pathRecovery, host), &struct{}{}) - // we ignore any zk errors here, as it will appear on next iteration - // and lead to lost state followed by recovery - return err == nil + return app.appDCS.IsRecoveryNeeded(host) } +// setResetupStatus is an unexported wrapper kept for internal callers (recovery.go). func (app *App) setResetupStatus(host string, status bool) error { - err := app.dcs.Create(pathResetupStatus, nil) - if err != nil && !errors.Is(err, dcs.ErrExists) { - return err - } - resetupStatus := &mysql.ResetupStatus{ - Status: status, - UpdateTime: time.Now(), - } - err = app.dcs.Set(dcs.JoinPath(pathResetupStatus, host), resetupStatus) - if err != nil { - return err - } - return nil + return app.appDCS.SetResetupStatus(host, status) } +// GetResetupStatus reads the resetup status for a host. func (app *App) GetResetupStatus(host string) (mysql.ResetupStatus, error) { - resetupStatus := mysql.ResetupStatus{} - err := app.dcs.Get(dcs.JoinPath(pathResetupStatus, host), &resetupStatus) - return resetupStatus, err + return app.appDCS.GetResetupStatus(host) } +// UpdateLastShutdownNodeTime records the current time as the last shutdown time. func (app *App) UpdateLastShutdownNodeTime() error { - err := app.dcs.Set(pathLastShutdownNodeTime, time.Now()) - if err != nil { - return err - } - return nil + return app.appDCS.UpdateLastShutdownNodeTime() } +// GetLastShutdownNodeTime returns the last recorded shutdown time. func (app *App) GetLastShutdownNodeTime() (time.Time, error) { - var t time.Time - err := app.dcs.Get(pathLastShutdownNodeTime, &t) - if errors.Is(err, dcs.ErrNotFound) { - err = app.dcs.Create(pathLastShutdownNodeTime, time.Now()) - if err != nil { - return time.Now(), err - } - return time.Now(), nil - } - return t, err + return app.appDCS.GetLastShutdownNodeTime() } -// FinishSwitchover finish current switchover and write the result +// FinishSwitchover finishes the current switchover and writes the result. +// Kept on *App because it calls timing methods (stopTiming, logSwitchoverFailure). func (app *App) FinishSwitchover(switchover *Switchover, switchErr error) error { result := true action := "finished" @@ -157,14 +135,18 @@ func (app *App) FinishSwitchover(switchover *Switchover, switchErr error) error app.stopTiming(timingFailover) } - err := app.dcs.Delete(pathCurrentSwitch) + err := app.appDCS.DeleteCurrentSwitchover() if err != nil { return err } - return app.dcs.Set(path, switchover) + if path == pathLastSwitch { + return app.appDCS.SetLastSwitchover(switchover) + } + return app.appDCS.SetLastRejectedSwitchover(switchover) } -// Fail current switchover, it will be repeated next cycle +// FailSwitchover marks the current switchover as failed (will be retried next cycle). +// Kept on *App for symmetry with FinishSwitchover and StartSwitchover. func (app *App) FailSwitchover(switchover *Switchover, err error) error { app.logger.Error().Err(err).Msgf("switchover: %s => %s failed", switchover.From, switchover.To) switchover.RunCount++ @@ -172,9 +154,11 @@ func (app *App) FailSwitchover(switchover *Switchover, err error) error { switchover.Result.Ok = false switchover.Result.Error = err.Error() switchover.Result.FinishedAt = time.Now() - return app.dcs.Set(pathCurrentSwitch, switchover) + return app.appDCS.SetCurrentSwitchover(switchover) } +// StartSwitchover records that a switchover has started. +// Kept on *App because it calls startTiming. func (app *App) StartSwitchover(switchover *Switchover) error { app.logger.Info().Msgf("switchover: %s => %s starting...", switchover.From, switchover.To) switchover.StartedAt = time.Now() @@ -182,78 +166,56 @@ func (app *App) StartSwitchover(switchover *Switchover) error { if switchover.MasterTransition == SwitchoverTransition { app.startTiming(timingSwitchover, switchover.InitiatedAt) } - return app.dcs.Set(pathCurrentSwitch, switchover) + return app.appDCS.SetCurrentSwitchover(switchover) } -func (app *App) GetLastSwitchover() Switchover { - var lastSwitch, lastRejectedSwitch Switchover - err := app.dcs.Get(pathLastSwitch, &lastSwitch) - if err != nil && !errors.Is(err, dcs.ErrNotFound) { - app.logger.Error().Err(err).Msg(pathLastSwitch) - } - errRejected := app.dcs.Get(pathLastRejectedSwitch, &lastRejectedSwitch) - if errRejected != nil && !errors.Is(errRejected, dcs.ErrNotFound) { - app.logger.Error().Err(errRejected).Msg(pathLastRejectedSwitch) - } +// GetCurrentSwitchover reads the current in-progress switchover from ZK. +// Returns dcs.ErrNotFound if no switchover is in progress. +func (app *App) GetCurrentSwitchover(switchover *Switchover) error { + return app.appDCS.GetCurrentSwitchover(switchover) +} - if lastRejectedSwitch.InitiatedAt.After(lastSwitch.InitiatedAt) { - return lastRejectedSwitch - } +// CreateCurrentSwitchover creates a new switchover record in ZK (fails if one already exists). +func (app *App) CreateCurrentSwitchover(switchover *Switchover) error { + return app.appDCS.CreateCurrentSwitchover(switchover) +} + +// DeleteCurrentSwitchover removes the current switchover node from ZK. +func (app *App) DeleteCurrentSwitchover() error { + return app.appDCS.DeleteCurrentSwitchover() +} - return lastSwitch +// GetLastSwitchover returns the most recent switchover (finished or rejected). +func (app *App) GetLastSwitchover() Switchover { + return app.appDCS.GetLastSwitchover() } +// IssueFailover creates a new failover switchover record in ZK. func (app *App) IssueFailover(master string) error { - var switchover Switchover - switchover.From = master - switchover.InitiatedBy = app.config.Hostname - switchover.InitiatedAt = time.Now() - switchover.Cause = CauseAuto - switchover.MasterTransition = FailoverTransition - return app.dcs.Create(pathCurrentSwitch, switchover) + return app.appDCS.IssueFailover(master) } +// SetMasterHost writes the current master hostname to ZK. func (app *App) SetMasterHost(master string) (string, error) { - err := app.dcs.Set(pathMasterNode, master) - if err != nil { - return "", fmt.Errorf("failed to set current master to dcs: %w", err) - } - return master, nil + return app.appDCS.SetMasterHost(master) } +// GetMasterHostFromDcs reads the current master hostname from ZK. func (app *App) GetMasterHostFromDcs() (string, error) { - var master string - err := app.dcs.Get(pathMasterNode, &master) - if err != nil && !errors.Is(err, dcs.ErrNotFound) { - return "", fmt.Errorf("failed to get current master from dcs: %w", err) - } - if master != "" { - return master, nil - } - return "", nil + return app.appDCS.GetMasterHostFromDcs() } +// SetReplMonTS writes the replication monitor timestamp to ZK. func (app *App) SetReplMonTS(ts string) error { - err := app.dcs.Create(pathMasterReplMonTS, ts) - if err != nil && !errors.Is(err, dcs.ErrExists) { - return err - } - err = app.dcs.Set(pathMasterReplMonTS, ts) - if err != nil { - return err - } - return nil + return app.appDCS.SetReplMonTS(ts) } +// GetReplMonTS reads the replication monitor timestamp from ZK. func (app *App) GetReplMonTS() (string, error) { - var ts string - err := app.dcs.Get(pathMasterReplMonTS, &ts) - if errors.Is(err, dcs.ErrNotFound) { - return "", nil - } - return ts, err + return app.appDCS.GetReplMonTS() } +// SetLowSpace writes the low-space flag to ZK. func (app *App) SetLowSpace(lowSpace bool) error { - return app.dcs.Set(pathLowSpace, lowSpace) + return app.appDCS.SetLowSpace(lowSpace) } diff --git a/internal/app/app_dcs_impl.go b/internal/app/app_dcs_impl.go new file mode 100644 index 00000000..20f710e5 --- /dev/null +++ b/internal/app/app_dcs_impl.go @@ -0,0 +1,308 @@ +package app + +import ( + "errors" + "fmt" + "time" + + nodestate "github.com/yandex/mysync/internal/app/node_state" + "github.com/yandex/mysync/internal/config" + "github.com/yandex/mysync/internal/dcs" + "github.com/yandex/mysync/internal/log" + "github.com/yandex/mysync/internal/mysql" + "github.com/yandex/mysync/internal/util" +) + +// appDCS implements IAppDCS by wrapping a low-level dcs.DCS. +// It owns all ZK path knowledge and serialization for mysync business logic. +// The struct is unexported; callers depend on the IAppDCS interface. +type appDCS struct { + dcs dcs.DCS + config *config.Config + logger *log.Logger +} + +// NewAppDCS creates a new appDCS. Returns IAppDCS so callers depend on the interface. +func NewAppDCS(d dcs.DCS, cfg *config.Config, logger *log.Logger) IAppDCS { + return &appDCS{dcs: d, config: cfg, logger: logger} +} + +// compile-time assertion +var _ IAppDCS = (*appDCS)(nil) + +// GetActiveNodes returns master + alive running replicas. +func (a *appDCS) GetActiveNodes() ([]string, error) { + var activeNodes []string + err := a.dcs.Get(pathActiveNodes, &activeNodes) + if err != nil { + if errors.Is(err, dcs.ErrNotFound) || errors.Is(err, dcs.ErrMalformed) { + return nil, nil + } + return nil, fmt.Errorf("failed to get active nodes from zk %w", err) + } + return activeNodes, nil +} + +// SetActiveNodes writes the active nodes list to ZK. +func (a *appDCS) SetActiveNodes(nodes []string) error { + return a.dcs.Set(pathActiveNodes, nodes) +} + +// DeleteActiveNodes removes the active nodes list from ZK. +func (a *appDCS) DeleteActiveNodes() error { + return a.dcs.Delete(pathActiveNodes) +} + +// SetHealthState writes the ephemeral per-host health state to ZK. +func (a *appDCS) SetHealthState(host string, state *nodestate.NodeState) error { + return a.dcs.SetEphemeral(dcs.JoinPath(pathHealthPrefix, host), state) +} + +// GetHealthState reads the per-host health state from ZK. +func (a *appDCS) GetHealthState(host string, state *nodestate.NodeState) error { + return a.dcs.Get(dcs.JoinPath(pathHealthPrefix, host), state) +} + +// SetMaintenance writes the maintenance record to ZK. +func (a *appDCS) SetMaintenance(maintenance *Maintenance) error { + return a.dcs.Set(pathMaintenance, maintenance) +} + +// DeleteMaintenance removes the maintenance record from ZK. +func (a *appDCS) DeleteMaintenance() error { + return a.dcs.Delete(pathMaintenance) +} + +// FetchCascadeNodeConfigurations reads all cascade node configurations from ZK. +func (a *appDCS) FetchCascadeNodeConfigurations() (map[string]mysql.CascadeNodeConfiguration, error) { + cascadeTopology := make(map[string]mysql.CascadeNodeConfiguration) + hosts, err := a.dcs.GetChildren(dcs.PathCascadeNodesPrefix) + if err != nil { + return cascadeTopology, err + } + for _, host := range hosts { + var cnc mysql.CascadeNodeConfiguration + err := a.dcs.Get(dcs.JoinPath(dcs.PathCascadeNodesPrefix, host), &cnc) + if err != nil { + return cascadeTopology, err + } + cascadeTopology[host] = cnc + } + return cascadeTopology, nil +} + +// GetNodeConfiguration reads the HA node configuration (priority etc.) from ZK. +func (a *appDCS) GetNodeConfiguration(host string) (mysql.NodeConfiguration, error) { + var nc mysql.NodeConfiguration + err := a.dcs.Get(dcs.JoinPath(pathHANodes, host), &nc) + return nc, err +} + +// GetClusterCascadeFqdnsFromDcs returns cascade node FQDNs stored in ZK. +func (a *appDCS) GetClusterCascadeFqdnsFromDcs() ([]string, error) { + fqdns, err := a.dcs.GetChildren(dcs.PathCascadeNodesPrefix) + if errors.Is(err, dcs.ErrNotFound) { + return make([]string, 0), nil + } + if err != nil { + return nil, err + } + return fqdns, nil +} + +// GetMaintenance returns the current maintenance record from ZK. +func (a *appDCS) GetMaintenance() (*Maintenance, error) { + maintenance := new(Maintenance) + err := a.dcs.Get(pathMaintenance, maintenance) + if err != nil { + return nil, err + } + return maintenance, err +} + +// GetHostsOnRecovery returns hosts currently marked for recovery. +func (a *appDCS) GetHostsOnRecovery() ([]string, error) { + hosts, err := a.dcs.GetChildren(pathRecovery) + if errors.Is(err, dcs.ErrNotFound) { + return nil, nil + } + return hosts, err +} + +// ClearRecovery removes the recovery marker for a host. +func (a *appDCS) ClearRecovery(host string) error { + return a.dcs.Delete(dcs.JoinPath(pathRecovery, host)) +} + +// SetRecovery marks a host for recovery and removes it from active nodes. +func (a *appDCS) SetRecovery(host string) error { + activeNodes, err := a.GetActiveNodes() + if err != nil { + return err + } + activeNodes = util.FilterStrings(activeNodes, func(n string) bool { + return n != host + }) + err = a.dcs.Set(pathActiveNodes, activeNodes) + if err != nil { + return err + } + err = a.dcs.Create(pathRecovery, nil) + if err != nil && !errors.Is(err, dcs.ErrExists) { + return err + } + err = a.dcs.Create(dcs.JoinPath(pathRecovery, host), nil) + if err != nil && !errors.Is(err, dcs.ErrExists) { + return err + } + return nil +} + +// IsRecoveryNeeded returns true if the host has a recovery marker in ZK. +func (a *appDCS) IsRecoveryNeeded(host string) bool { + err := a.dcs.Get(dcs.JoinPath(pathRecovery, host), &struct{}{}) + // we ignore any zk errors here, as it will appear on next iteration + // and lead to lost state followed by recovery + return err == nil +} + +// SetResetupStatus writes the resetup status for a host. +func (a *appDCS) SetResetupStatus(host string, status bool) error { + err := a.dcs.Create(pathResetupStatus, nil) + if err != nil && !errors.Is(err, dcs.ErrExists) { + return err + } + resetupStatus := &mysql.ResetupStatus{ + Status: status, + UpdateTime: time.Now(), + } + return a.dcs.Set(dcs.JoinPath(pathResetupStatus, host), resetupStatus) +} + +// GetResetupStatus reads the resetup status for a host. +func (a *appDCS) GetResetupStatus(host string) (mysql.ResetupStatus, error) { + resetupStatus := mysql.ResetupStatus{} + err := a.dcs.Get(dcs.JoinPath(pathResetupStatus, host), &resetupStatus) + return resetupStatus, err +} + +// UpdateLastShutdownNodeTime records the current time as the last shutdown time. +func (a *appDCS) UpdateLastShutdownNodeTime() error { + return a.dcs.Set(pathLastShutdownNodeTime, time.Now()) +} + +// GetLastShutdownNodeTime returns the last recorded shutdown time, creating it if absent. +func (a *appDCS) GetLastShutdownNodeTime() (time.Time, error) { + var t time.Time + err := a.dcs.Get(pathLastShutdownNodeTime, &t) + if errors.Is(err, dcs.ErrNotFound) { + err = a.dcs.Create(pathLastShutdownNodeTime, time.Now()) + if err != nil { + return time.Now(), err + } + return time.Now(), nil + } + return t, err +} + +// GetLastSwitchover returns the most recent switchover (finished or rejected). +func (a *appDCS) GetLastSwitchover() Switchover { + var lastSwitch, lastRejectedSwitch Switchover + err := a.dcs.Get(pathLastSwitch, &lastSwitch) + if err != nil && !errors.Is(err, dcs.ErrNotFound) { + a.logger.Error().Err(err).Msg(pathLastSwitch) + } + errRejected := a.dcs.Get(pathLastRejectedSwitch, &lastRejectedSwitch) + if errRejected != nil && !errors.Is(errRejected, dcs.ErrNotFound) { + a.logger.Error().Err(errRejected).Msg(pathLastRejectedSwitch) + } + if lastRejectedSwitch.InitiatedAt.After(lastSwitch.InitiatedAt) { + return lastRejectedSwitch + } + return lastSwitch +} + +// GetCurrentSwitchover reads the current in-progress switchover from ZK. +// Returns dcs.ErrNotFound if no switchover is in progress. +func (a *appDCS) GetCurrentSwitchover(switchover *Switchover) error { + return a.dcs.Get(pathCurrentSwitch, switchover) +} + +// CreateCurrentSwitchover creates a new switchover record in ZK (fails if one already exists). +func (a *appDCS) CreateCurrentSwitchover(switchover *Switchover) error { + return a.dcs.Create(pathCurrentSwitch, switchover) +} + +// SetCurrentSwitchover writes the current in-progress switchover to ZK. +func (a *appDCS) SetCurrentSwitchover(switchover *Switchover) error { + return a.dcs.Set(pathCurrentSwitch, switchover) +} + +// DeleteCurrentSwitchover removes the current switchover node from ZK. +func (a *appDCS) DeleteCurrentSwitchover() error { + return a.dcs.Delete(pathCurrentSwitch) +} + +// SetLastSwitchover writes the completed switchover result to ZK. +func (a *appDCS) SetLastSwitchover(switchover *Switchover) error { + return a.dcs.Set(pathLastSwitch, switchover) +} + +// SetLastRejectedSwitchover writes the rejected switchover result to ZK. +func (a *appDCS) SetLastRejectedSwitchover(switchover *Switchover) error { + return a.dcs.Set(pathLastRejectedSwitch, switchover) +} + +// IssueFailover creates a new failover switchover record in ZK. +func (a *appDCS) IssueFailover(master string) error { + var switchover Switchover + switchover.From = master + switchover.InitiatedBy = a.config.Hostname + switchover.InitiatedAt = time.Now() + switchover.Cause = CauseAuto + switchover.MasterTransition = FailoverTransition + return a.dcs.Create(pathCurrentSwitch, switchover) +} + +// SetMasterHost writes the current master hostname to ZK. +func (a *appDCS) SetMasterHost(master string) (string, error) { + err := a.dcs.Set(pathMasterNode, master) + if err != nil { + return "", fmt.Errorf("failed to set current master to dcs: %w", err) + } + return master, nil +} + +// GetMasterHostFromDcs reads the current master hostname from ZK. +func (a *appDCS) GetMasterHostFromDcs() (string, error) { + var master string + err := a.dcs.Get(pathMasterNode, &master) + if err != nil && !errors.Is(err, dcs.ErrNotFound) { + return "", fmt.Errorf("failed to get current master from dcs: %w", err) + } + return master, nil +} + +// SetReplMonTS writes the replication monitor timestamp to ZK. +func (a *appDCS) SetReplMonTS(ts string) error { + err := a.dcs.Create(pathMasterReplMonTS, ts) + if err != nil && !errors.Is(err, dcs.ErrExists) { + return err + } + return a.dcs.Set(pathMasterReplMonTS, ts) +} + +// GetReplMonTS reads the replication monitor timestamp from ZK. +func (a *appDCS) GetReplMonTS() (string, error) { + var ts string + err := a.dcs.Get(pathMasterReplMonTS, &ts) + if errors.Is(err, dcs.ErrNotFound) { + return "", nil + } + return ts, err +} + +// SetLowSpace writes the low-space flag to ZK. +func (a *appDCS) SetLowSpace(lowSpace bool) error { + return a.dcs.Set(pathLowSpace, lowSpace) +} diff --git a/internal/app/appdcs_test.go b/internal/app/appdcs_test.go new file mode 100644 index 00000000..dddcd205 --- /dev/null +++ b/internal/app/appdcs_test.go @@ -0,0 +1,309 @@ +package app + +import ( + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + nodestate "github.com/yandex/mysync/internal/app/node_state" + "github.com/yandex/mysync/internal/config" + "github.com/yandex/mysync/internal/log" + "github.com/yandex/mysync/internal/mysql" +) + +// newTestApp builds a minimal *App suitable for unit tests. +// appDCS is injected so tests can pass a MockIAppDCS. +func newTestApp(t *testing.T, cfg *config.Config, appDCS IAppDCS) *App { + t.Helper() + logger, _, _, err := log.Open("/dev/null", "fatal", 100, 0) + require.NoError(t, err) + return &App{ + config: cfg, + logger: logger, + t: NewTimings(), + appDCS: appDCS, + switchHelper: mysql.NewSwitchHelper(cfg), + } +} + +// minConfig returns a config with sensible defaults for tests. +func minConfig() *config.Config { + return &config.Config{ + Failover: true, + FailoverCooldown: 30 * time.Second, + FailoverDelay: 0, + SemiSync: false, + } +} + +// aliveReplica returns a NodeState for a healthy HA replica. +// SlaveState must be non-nil for countAliveHASlavesWithinNodes to count it. +func aliveReplica() *nodestate.NodeState { + return &nodestate.NodeState{ + PingOk: true, + IsMaster: false, + IsCascade: false, + SlaveState: &nodestate.SlaveState{}, + } +} + +// clusterState builds a map[host]NodeState with the given master and HA replicas. +// +//nolint:unparam +func clusterState(master string, replicas ...string) map[string]*nodestate.NodeState { + m := make(map[string]*nodestate.NodeState) + m[master] = &nodestate.NodeState{PingOk: true, IsMaster: true} + for _, r := range replicas { + m[r] = aliveReplica() + } + return m +} + +// ─── approveFailover ──────────────────────────────────────────────────────── + +func TestApproveFailover_DisabledInConfig(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cfg := minConfig() + cfg.Failover = false + app := newTestApp(t, cfg, NewMockIAppDCS(ctrl)) + + cs := clusterState("master", "replica1") + err := app.approveFailover(cs, cs, []string{"replica1"}, "master") + require.ErrorContains(t, err, "auto_failover is disabled") +} + +func TestApproveFailover_NoQuorum(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cfg := minConfig() + cfg.SemiSync = true + cfg.RplSemiSyncMasterWaitForSlaveCount = 1 + + mockDCS := NewMockIAppDCS(ctrl) + // GetLastSwitchover is called after quorum check — but quorum fails first, so no call expected + app := newTestApp(t, cfg, mockDCS) + + cs := clusterState("master") // no replicas → no quorum + err := app.approveFailover(cs, cs, []string{}, "master") + require.ErrorContains(t, err, "no quorum") +} + +func TestApproveFailover_CooldownNotElapsed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cfg := minConfig() + cfg.FailoverCooldown = 60 * time.Second + + mockDCS := NewMockIAppDCS(ctrl) + // Last auto-failover finished 10 seconds ago — within cooldown + lastSwitch := Switchover{ + Cause: CauseAuto, + Result: &SwitchoverResult{ + Ok: true, + FinishedAt: time.Now().Add(-10 * time.Second), + }, + } + mockDCS.EXPECT().GetLastSwitchover().Return(lastSwitch) + + app := newTestApp(t, cfg, mockDCS) + + cs := clusterState("master", "replica1") + err := app.approveFailover(cs, cs, []string{"replica1"}, "master") + require.ErrorContains(t, err, "not enough time from last failover") +} + +func TestApproveFailover_CooldownElapsed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cfg := minConfig() + cfg.FailoverCooldown = 10 * time.Second + + mockDCS := NewMockIAppDCS(ctrl) + // Last auto-failover finished 60 seconds ago — cooldown elapsed + lastSwitch := Switchover{ + Cause: CauseAuto, + Result: &SwitchoverResult{ + Ok: true, + FinishedAt: time.Now().Add(-60 * time.Second), + }, + } + mockDCS.EXPECT().GetLastSwitchover().Return(lastSwitch) + + app := newTestApp(t, cfg, mockDCS) + + cs := clusterState("master", "replica1") + err := app.approveFailover(cs, cs, []string{"replica1"}, "master") + require.NoError(t, err) +} + +func TestApproveFailover_ManualSwitchoverDoesNotTriggerCooldown(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cfg := minConfig() + cfg.FailoverCooldown = 60 * time.Second + + mockDCS := NewMockIAppDCS(ctrl) + // Last switchover was manual (CauseManual) — cooldown only applies to CauseAuto + lastSwitch := Switchover{ + Cause: CauseManual, + Result: &SwitchoverResult{ + Ok: true, + FinishedAt: time.Now().Add(-5 * time.Second), + }, + } + mockDCS.EXPECT().GetLastSwitchover().Return(lastSwitch) + + app := newTestApp(t, cfg, mockDCS) + + cs := clusterState("master", "replica1") + err := app.approveFailover(cs, cs, []string{"replica1"}, "master") + require.NoError(t, err) +} + +func TestApproveFailover_NoLastSwitchover(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cfg := minConfig() + + mockDCS := NewMockIAppDCS(ctrl) + // No previous switchover — zero-value Switchover has nil Result + mockDCS.EXPECT().GetLastSwitchover().Return(Switchover{}) + + app := newTestApp(t, cfg, mockDCS) + + cs := clusterState("master", "replica1") + err := app.approveFailover(cs, cs, []string{"replica1"}, "master") + require.NoError(t, err) +} + +func TestApproveFailover_AfterCrashRecovery_SkipsDelayCheck(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cfg := minConfig() + cfg.FailoverDelay = 60 * time.Second // would normally block + cfg.ResetupCrashedHosts = true + + mockDCS := NewMockIAppDCS(ctrl) + mockDCS.EXPECT().GetLastSwitchover().Return(Switchover{}) + + app := newTestApp(t, cfg, mockDCS) + // NodeFailedAt is zero → failingTime is huge, but crash recovery skips the check + cs := clusterState("master", "replica1") + dcsState := map[string]*nodestate.NodeState{ + "master": { + PingOk: true, + IsMaster: true, + DaemonState: &nodestate.DaemonState{ + CrashRecovery: true, + }, + }, + "replica1": {PingOk: true}, + } + + err := app.approveFailover(cs, dcsState, []string{"replica1"}, "master") + require.NoError(t, err) +} + +// ─── getMasterHost ─────────────────────────────────────────────────────────── + +func TestGetMasterHost_OneMaster(t *testing.T) { + app := newTestApp(t, minConfig(), nil) + cs := map[string]*nodestate.NodeState{ + "master": {PingOk: true, IsMaster: true}, + "replica1": aliveReplica(), + "replica2": aliveReplica(), + } + host, err := app.getMasterHost(cs) + require.NoError(t, err) + require.Equal(t, "master", host) +} + +func TestGetMasterHost_NoMaster(t *testing.T) { + app := newTestApp(t, minConfig(), nil) + cs := map[string]*nodestate.NodeState{ + "node1": {PingOk: true, IsMaster: false}, + "node2": {PingOk: true, IsMaster: false}, + } + host, err := app.getMasterHost(cs) + require.NoError(t, err) + require.Empty(t, host) +} + +func TestGetMasterHost_ManyMasters(t *testing.T) { + app := newTestApp(t, minConfig(), nil) + cs := map[string]*nodestate.NodeState{ + "node1": {PingOk: true, IsMaster: true}, + "node2": {PingOk: true, IsMaster: true}, + } + _, err := app.getMasterHost(cs) + require.ErrorIs(t, err, ErrManyMasters) +} + +func TestGetMasterHost_DeadMasterNotCounted(t *testing.T) { + // A node with PingOk=false and IsMaster=true should NOT be returned as master + app := newTestApp(t, minConfig(), nil) + cs := map[string]*nodestate.NodeState{ + "dead-master": {PingOk: false, IsMaster: true}, + "replica1": {PingOk: true, IsMaster: false}, + } + host, err := app.getMasterHost(cs) + require.NoError(t, err) + require.Empty(t, host) +} + +// ─── getCurrentMaster ──────────────────────────────────────────────────────── + +func TestGetCurrentMaster_FromDCS(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDCS := NewMockIAppDCS(ctrl) + mockDCS.EXPECT().GetMasterHostFromDcs().Return("master", nil) + + app := newTestApp(t, minConfig(), mockDCS) + cs := clusterState("master", "replica1") + host, err := app.getCurrentMaster(cs) + require.NoError(t, err) + require.Equal(t, "master", host) +} + +func TestGetCurrentMaster_FallbackToClusterState(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDCS := NewMockIAppDCS(ctrl) + // DCS returns empty — fall back to scanning cluster state + mockDCS.EXPECT().GetMasterHostFromDcs().Return("", nil) + mockDCS.EXPECT().SetMasterHost("master").Return("master", nil) + + app := newTestApp(t, minConfig(), mockDCS) + cs := clusterState("master", "replica1") + host, err := app.getCurrentMaster(cs) + require.NoError(t, err) + require.Equal(t, "master", host) +} + +func TestGetCurrentMaster_NoMasterAnywhere(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDCS := NewMockIAppDCS(ctrl) + mockDCS.EXPECT().GetMasterHostFromDcs().Return("", nil) + + app := newTestApp(t, minConfig(), mockDCS) + cs := map[string]*nodestate.NodeState{ + "node1": {PingOk: true, IsMaster: false}, + } + _, err := app.getCurrentMaster(cs) + require.ErrorIs(t, err, ErrNoMaster) +} diff --git a/internal/app/cli_switch.go b/internal/app/cli_switch.go index 92824f0a..6b3754db 100644 --- a/internal/app/cli_switch.go +++ b/internal/app/cli_switch.go @@ -115,7 +115,7 @@ func (app *App) CliSwitch(switchFrom, switchTo string, waitTimeout time.Duration } var switchover Switchover - err = app.dcs.Get(pathCurrentSwitch, &switchover) + err = app.GetCurrentSwitchover(&switchover) if err == nil { app.logger.Error().Msgf("Another switchover in progress %v", switchover) return 2 @@ -136,7 +136,7 @@ func (app *App) CliSwitch(switchFrom, switchTo string, waitTimeout time.Duration switchover.MasterTransition = SwitchoverTransition } - err = app.dcs.Create(pathCurrentSwitch, switchover) + err = app.CreateCurrentSwitchover(&switchover) if errors.Is(err, dcs.ErrExists) { app.logger.Error().Msg("Another switchover in progress") return 2 @@ -189,7 +189,7 @@ func (app *App) CliAbort() int { defer app.dcs.Close() app.dcs.Initialize() - err = app.dcs.Get(pathCurrentSwitch, new(Switchover)) + err = app.GetCurrentSwitchover(new(Switchover)) if errors.Is(err, dcs.ErrNotFound) { fmt.Println("no active switchover") return 0 @@ -212,7 +212,7 @@ func (app *App) CliAbort() int { return 1 } - err = app.dcs.Delete(pathCurrentSwitch) + err = app.DeleteCurrentSwitchover() if err != nil { app.logger.Error().Err(err).Msg("") return 1 diff --git a/internal/app/idcs.go b/internal/app/idcs.go new file mode 100644 index 00000000..29ceba4c --- /dev/null +++ b/internal/app/idcs.go @@ -0,0 +1,72 @@ +//go:generate mockgen -source=idcs.go -destination=mock_idcs_test.go -package=app . IAppDCS +package app + +import ( + "time" + + nodestate "github.com/yandex/mysync/internal/app/node_state" + "github.com/yandex/mysync/internal/mysql" +) + +// IAppDCS is a high-level interface for mysync ZooKeeper operations. +// It encapsulates all ZK paths and serialization, hiding the low-level dcs.DCS. +// Note: FinishSwitchover, StartSwitchover, FailSwitchover are NOT part of this +// interface because they call timing methods (startTiming/stopTiming/logSwitchoverFailure) +// defined on *App. They remain as *App methods that delegate to appDCS for pure ZK ops. +type IAppDCS interface { + // Active nodes + GetActiveNodes() ([]string, error) + SetActiveNodes(nodes []string) error + DeleteActiveNodes() error + + // Master + GetMasterHostFromDcs() (string, error) + SetMasterHost(master string) (string, error) + + // Health state (ephemeral per-host node state written by healthChecker) + SetHealthState(host string, state *nodestate.NodeState) error + GetHealthState(host string, state *nodestate.NodeState) error + + // Maintenance + GetMaintenance() (*Maintenance, error) + SetMaintenance(maintenance *Maintenance) error + DeleteMaintenance() error + + // Recovery + GetHostsOnRecovery() ([]string, error) + SetRecovery(host string) error + ClearRecovery(host string) error + IsRecoveryNeeded(host string) bool + + // Resetup + GetResetupStatus(host string) (mysql.ResetupStatus, error) + SetResetupStatus(host string, status bool) error + + // Switchover state (pure ZK ops, no timing side-effects) + GetCurrentSwitchover(switchover *Switchover) error + CreateCurrentSwitchover(switchover *Switchover) error + GetLastSwitchover() Switchover + SetCurrentSwitchover(switchover *Switchover) error + DeleteCurrentSwitchover() error + SetLastSwitchover(switchover *Switchover) error + SetLastRejectedSwitchover(switchover *Switchover) error + IssueFailover(master string) error + + // Shutdown tracking + GetLastShutdownNodeTime() (time.Time, error) + UpdateLastShutdownNodeTime() error + + // Misc + SetLowSpace(lowSpace bool) error + + // ReplMon + GetReplMonTS() (string, error) + SetReplMonTS(ts string) error + + // Cascade nodes + GetClusterCascadeFqdnsFromDcs() ([]string, error) + FetchCascadeNodeConfigurations() (map[string]mysql.CascadeNodeConfiguration, error) + + // HA node configuration + GetNodeConfiguration(host string) (mysql.NodeConfiguration, error) +} diff --git a/internal/app/mock_idcs_test.go b/internal/app/mock_idcs_test.go new file mode 100644 index 00000000..25c15f8a --- /dev/null +++ b/internal/app/mock_idcs_test.go @@ -0,0 +1,496 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/app/idcs.go + +// Package app is a generated GoMock package. +package app + +import ( + reflect "reflect" + time "time" + + gomock "github.com/golang/mock/gomock" + node_state "github.com/yandex/mysync/internal/app/node_state" + mysql "github.com/yandex/mysync/internal/mysql" +) + +// MockIAppDCS is a mock of IAppDCS interface. +type MockIAppDCS struct { + ctrl *gomock.Controller + recorder *MockIAppDCSMockRecorder +} + +// MockIAppDCSMockRecorder is the mock recorder for MockIAppDCS. +type MockIAppDCSMockRecorder struct { + mock *MockIAppDCS +} + +// NewMockIAppDCS creates a new mock instance. +func NewMockIAppDCS(ctrl *gomock.Controller) *MockIAppDCS { + mock := &MockIAppDCS{ctrl: ctrl} + mock.recorder = &MockIAppDCSMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIAppDCS) EXPECT() *MockIAppDCSMockRecorder { + return m.recorder +} + +// ClearRecovery mocks base method. +func (m *MockIAppDCS) ClearRecovery(host string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClearRecovery", host) + ret0, _ := ret[0].(error) + return ret0 +} + +// ClearRecovery indicates an expected call of ClearRecovery. +func (mr *MockIAppDCSMockRecorder) ClearRecovery(host interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearRecovery", reflect.TypeOf((*MockIAppDCS)(nil).ClearRecovery), host) +} + +// CreateCurrentSwitchover mocks base method. +func (m *MockIAppDCS) CreateCurrentSwitchover(switchover *Switchover) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateCurrentSwitchover", switchover) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateCurrentSwitchover indicates an expected call of CreateCurrentSwitchover. +func (mr *MockIAppDCSMockRecorder) CreateCurrentSwitchover(switchover interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCurrentSwitchover", reflect.TypeOf((*MockIAppDCS)(nil).CreateCurrentSwitchover), switchover) +} + +// DeleteActiveNodes mocks base method. +func (m *MockIAppDCS) DeleteActiveNodes() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteActiveNodes") + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteActiveNodes indicates an expected call of DeleteActiveNodes. +func (mr *MockIAppDCSMockRecorder) DeleteActiveNodes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteActiveNodes", reflect.TypeOf((*MockIAppDCS)(nil).DeleteActiveNodes)) +} + +// DeleteCurrentSwitchover mocks base method. +func (m *MockIAppDCS) DeleteCurrentSwitchover() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCurrentSwitchover") + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteCurrentSwitchover indicates an expected call of DeleteCurrentSwitchover. +func (mr *MockIAppDCSMockRecorder) DeleteCurrentSwitchover() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCurrentSwitchover", reflect.TypeOf((*MockIAppDCS)(nil).DeleteCurrentSwitchover)) +} + +// DeleteMaintenance mocks base method. +func (m *MockIAppDCS) DeleteMaintenance() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteMaintenance") + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteMaintenance indicates an expected call of DeleteMaintenance. +func (mr *MockIAppDCSMockRecorder) DeleteMaintenance() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMaintenance", reflect.TypeOf((*MockIAppDCS)(nil).DeleteMaintenance)) +} + +// FetchCascadeNodeConfigurations mocks base method. +func (m *MockIAppDCS) FetchCascadeNodeConfigurations() (map[string]mysql.CascadeNodeConfiguration, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FetchCascadeNodeConfigurations") + ret0, _ := ret[0].(map[string]mysql.CascadeNodeConfiguration) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FetchCascadeNodeConfigurations indicates an expected call of FetchCascadeNodeConfigurations. +func (mr *MockIAppDCSMockRecorder) FetchCascadeNodeConfigurations() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchCascadeNodeConfigurations", reflect.TypeOf((*MockIAppDCS)(nil).FetchCascadeNodeConfigurations)) +} + +// GetActiveNodes mocks base method. +func (m *MockIAppDCS) GetActiveNodes() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetActiveNodes") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetActiveNodes indicates an expected call of GetActiveNodes. +func (mr *MockIAppDCSMockRecorder) GetActiveNodes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveNodes", reflect.TypeOf((*MockIAppDCS)(nil).GetActiveNodes)) +} + +// GetClusterCascadeFqdnsFromDcs mocks base method. +func (m *MockIAppDCS) GetClusterCascadeFqdnsFromDcs() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClusterCascadeFqdnsFromDcs") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetClusterCascadeFqdnsFromDcs indicates an expected call of GetClusterCascadeFqdnsFromDcs. +func (mr *MockIAppDCSMockRecorder) GetClusterCascadeFqdnsFromDcs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterCascadeFqdnsFromDcs", reflect.TypeOf((*MockIAppDCS)(nil).GetClusterCascadeFqdnsFromDcs)) +} + +// GetCurrentSwitchover mocks base method. +func (m *MockIAppDCS) GetCurrentSwitchover(switchover *Switchover) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCurrentSwitchover", switchover) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetCurrentSwitchover indicates an expected call of GetCurrentSwitchover. +func (mr *MockIAppDCSMockRecorder) GetCurrentSwitchover(switchover interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentSwitchover", reflect.TypeOf((*MockIAppDCS)(nil).GetCurrentSwitchover), switchover) +} + +// GetHealthState mocks base method. +func (m *MockIAppDCS) GetHealthState(host string, state *node_state.NodeState) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHealthState", host, state) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetHealthState indicates an expected call of GetHealthState. +func (mr *MockIAppDCSMockRecorder) GetHealthState(host, state interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHealthState", reflect.TypeOf((*MockIAppDCS)(nil).GetHealthState), host, state) +} + +// GetHostsOnRecovery mocks base method. +func (m *MockIAppDCS) GetHostsOnRecovery() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHostsOnRecovery") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetHostsOnRecovery indicates an expected call of GetHostsOnRecovery. +func (mr *MockIAppDCSMockRecorder) GetHostsOnRecovery() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHostsOnRecovery", reflect.TypeOf((*MockIAppDCS)(nil).GetHostsOnRecovery)) +} + +// GetLastShutdownNodeTime mocks base method. +func (m *MockIAppDCS) GetLastShutdownNodeTime() (time.Time, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLastShutdownNodeTime") + ret0, _ := ret[0].(time.Time) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLastShutdownNodeTime indicates an expected call of GetLastShutdownNodeTime. +func (mr *MockIAppDCSMockRecorder) GetLastShutdownNodeTime() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastShutdownNodeTime", reflect.TypeOf((*MockIAppDCS)(nil).GetLastShutdownNodeTime)) +} + +// GetLastSwitchover mocks base method. +func (m *MockIAppDCS) GetLastSwitchover() Switchover { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLastSwitchover") + ret0, _ := ret[0].(Switchover) + return ret0 +} + +// GetLastSwitchover indicates an expected call of GetLastSwitchover. +func (mr *MockIAppDCSMockRecorder) GetLastSwitchover() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastSwitchover", reflect.TypeOf((*MockIAppDCS)(nil).GetLastSwitchover)) +} + +// GetMaintenance mocks base method. +func (m *MockIAppDCS) GetMaintenance() (*Maintenance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaintenance") + ret0, _ := ret[0].(*Maintenance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMaintenance indicates an expected call of GetMaintenance. +func (mr *MockIAppDCSMockRecorder) GetMaintenance() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaintenance", reflect.TypeOf((*MockIAppDCS)(nil).GetMaintenance)) +} + +// GetMasterHostFromDcs mocks base method. +func (m *MockIAppDCS) GetMasterHostFromDcs() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMasterHostFromDcs") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMasterHostFromDcs indicates an expected call of GetMasterHostFromDcs. +func (mr *MockIAppDCSMockRecorder) GetMasterHostFromDcs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMasterHostFromDcs", reflect.TypeOf((*MockIAppDCS)(nil).GetMasterHostFromDcs)) +} + +// GetNodeConfiguration mocks base method. +func (m *MockIAppDCS) GetNodeConfiguration(host string) (mysql.NodeConfiguration, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNodeConfiguration", host) + ret0, _ := ret[0].(mysql.NodeConfiguration) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNodeConfiguration indicates an expected call of GetNodeConfiguration. +func (mr *MockIAppDCSMockRecorder) GetNodeConfiguration(host interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeConfiguration", reflect.TypeOf((*MockIAppDCS)(nil).GetNodeConfiguration), host) +} + +// GetReplMonTS mocks base method. +func (m *MockIAppDCS) GetReplMonTS() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetReplMonTS") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetReplMonTS indicates an expected call of GetReplMonTS. +func (mr *MockIAppDCSMockRecorder) GetReplMonTS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetReplMonTS", reflect.TypeOf((*MockIAppDCS)(nil).GetReplMonTS)) +} + +// GetResetupStatus mocks base method. +func (m *MockIAppDCS) GetResetupStatus(host string) (mysql.ResetupStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetResetupStatus", host) + ret0, _ := ret[0].(mysql.ResetupStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetResetupStatus indicates an expected call of GetResetupStatus. +func (mr *MockIAppDCSMockRecorder) GetResetupStatus(host interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResetupStatus", reflect.TypeOf((*MockIAppDCS)(nil).GetResetupStatus), host) +} + +// IsRecoveryNeeded mocks base method. +func (m *MockIAppDCS) IsRecoveryNeeded(host string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsRecoveryNeeded", host) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsRecoveryNeeded indicates an expected call of IsRecoveryNeeded. +func (mr *MockIAppDCSMockRecorder) IsRecoveryNeeded(host interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRecoveryNeeded", reflect.TypeOf((*MockIAppDCS)(nil).IsRecoveryNeeded), host) +} + +// IssueFailover mocks base method. +func (m *MockIAppDCS) IssueFailover(master string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IssueFailover", master) + ret0, _ := ret[0].(error) + return ret0 +} + +// IssueFailover indicates an expected call of IssueFailover. +func (mr *MockIAppDCSMockRecorder) IssueFailover(master interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssueFailover", reflect.TypeOf((*MockIAppDCS)(nil).IssueFailover), master) +} + +// SetActiveNodes mocks base method. +func (m *MockIAppDCS) SetActiveNodes(nodes []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetActiveNodes", nodes) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetActiveNodes indicates an expected call of SetActiveNodes. +func (mr *MockIAppDCSMockRecorder) SetActiveNodes(nodes interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetActiveNodes", reflect.TypeOf((*MockIAppDCS)(nil).SetActiveNodes), nodes) +} + +// SetCurrentSwitchover mocks base method. +func (m *MockIAppDCS) SetCurrentSwitchover(switchover *Switchover) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetCurrentSwitchover", switchover) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetCurrentSwitchover indicates an expected call of SetCurrentSwitchover. +func (mr *MockIAppDCSMockRecorder) SetCurrentSwitchover(switchover interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCurrentSwitchover", reflect.TypeOf((*MockIAppDCS)(nil).SetCurrentSwitchover), switchover) +} + +// SetHealthState mocks base method. +func (m *MockIAppDCS) SetHealthState(host string, state *node_state.NodeState) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetHealthState", host, state) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetHealthState indicates an expected call of SetHealthState. +func (mr *MockIAppDCSMockRecorder) SetHealthState(host, state interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHealthState", reflect.TypeOf((*MockIAppDCS)(nil).SetHealthState), host, state) +} + +// SetLastRejectedSwitchover mocks base method. +func (m *MockIAppDCS) SetLastRejectedSwitchover(switchover *Switchover) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLastRejectedSwitchover", switchover) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLastRejectedSwitchover indicates an expected call of SetLastRejectedSwitchover. +func (mr *MockIAppDCSMockRecorder) SetLastRejectedSwitchover(switchover interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastRejectedSwitchover", reflect.TypeOf((*MockIAppDCS)(nil).SetLastRejectedSwitchover), switchover) +} + +// SetLastSwitchover mocks base method. +func (m *MockIAppDCS) SetLastSwitchover(switchover *Switchover) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLastSwitchover", switchover) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLastSwitchover indicates an expected call of SetLastSwitchover. +func (mr *MockIAppDCSMockRecorder) SetLastSwitchover(switchover interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastSwitchover", reflect.TypeOf((*MockIAppDCS)(nil).SetLastSwitchover), switchover) +} + +// SetLowSpace mocks base method. +func (m *MockIAppDCS) SetLowSpace(lowSpace bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLowSpace", lowSpace) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLowSpace indicates an expected call of SetLowSpace. +func (mr *MockIAppDCSMockRecorder) SetLowSpace(lowSpace interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLowSpace", reflect.TypeOf((*MockIAppDCS)(nil).SetLowSpace), lowSpace) +} + +// SetMaintenance mocks base method. +func (m *MockIAppDCS) SetMaintenance(maintenance *Maintenance) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetMaintenance", maintenance) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetMaintenance indicates an expected call of SetMaintenance. +func (mr *MockIAppDCSMockRecorder) SetMaintenance(maintenance interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetMaintenance", reflect.TypeOf((*MockIAppDCS)(nil).SetMaintenance), maintenance) +} + +// SetMasterHost mocks base method. +func (m *MockIAppDCS) SetMasterHost(master string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetMasterHost", master) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SetMasterHost indicates an expected call of SetMasterHost. +func (mr *MockIAppDCSMockRecorder) SetMasterHost(master interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetMasterHost", reflect.TypeOf((*MockIAppDCS)(nil).SetMasterHost), master) +} + +// SetRecovery mocks base method. +func (m *MockIAppDCS) SetRecovery(host string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetRecovery", host) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetRecovery indicates an expected call of SetRecovery. +func (mr *MockIAppDCSMockRecorder) SetRecovery(host interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetRecovery", reflect.TypeOf((*MockIAppDCS)(nil).SetRecovery), host) +} + +// SetReplMonTS mocks base method. +func (m *MockIAppDCS) SetReplMonTS(ts string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetReplMonTS", ts) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetReplMonTS indicates an expected call of SetReplMonTS. +func (mr *MockIAppDCSMockRecorder) SetReplMonTS(ts interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetReplMonTS", reflect.TypeOf((*MockIAppDCS)(nil).SetReplMonTS), ts) +} + +// SetResetupStatus mocks base method. +func (m *MockIAppDCS) SetResetupStatus(host string, status bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetResetupStatus", host, status) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetResetupStatus indicates an expected call of SetResetupStatus. +func (mr *MockIAppDCSMockRecorder) SetResetupStatus(host, status interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetResetupStatus", reflect.TypeOf((*MockIAppDCS)(nil).SetResetupStatus), host, status) +} + +// UpdateLastShutdownNodeTime mocks base method. +func (m *MockIAppDCS) UpdateLastShutdownNodeTime() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateLastShutdownNodeTime") + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLastShutdownNodeTime indicates an expected call of UpdateLastShutdownNodeTime. +func (mr *MockIAppDCSMockRecorder) UpdateLastShutdownNodeTime() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLastShutdownNodeTime", reflect.TypeOf((*MockIAppDCS)(nil).UpdateLastShutdownNodeTime)) +}