From 293e56bcde5bf452977e76dd985abf976e9b716e Mon Sep 17 00:00:00 2001 From: virant Date: Mon, 2 Nov 2026 15:44:56 +0100 Subject: [PATCH] fix(updates): clear superseded install failures; don't let one source block installs The WTR Max's Updates page still said "Installing updates failed (2026-11-01T15:36:28Z)" a day later. That install, started from the page, failed because apt.bcachefs.org refused connections; unattended upgrades had since installed kernel 7.2.15-84 and OpenZFS without an error, but the page only reads the last run started from it. - A failed install older than /var/lib/apt/periodic/upgrade-stamp, which apt.systemd.daily touches only when unattended-upgrade exits 0, is reported as superseded: the page shows it as history, with when updates last installed. Failed checks are not cleared this way: the unattended refresh runs in apt's default mode, which exits 0 even with an unreachable archive. - An install no longer stops when a package source can't be refreshed. One third-party archive being down kept SmoothNAS's own updates from installing. The refresh still runs with Error-Mode=any, its result is logged on its own line, and the install runs regardless. An install that succeeded with a failed refresh says so, and lists the sources. Tests: - Go: the WTR Max log superseded by a newer upgrade-stamp; not superseded by an older or same-second stamp, a missing stamp, a failed check, a successful or unfinished install. The generated install script is run under /bin/sh with stub apt-get/dpkg: a failed refresh still installs and is recorded, a failed install keeps its exit code, a clean run reports nothing. Mutation-checked. - Browser: the superseded WTR Max failure shows no error, the same failure without a later success still does (negative control), and a successful install with a failed refresh shows its warning. --- tests/browser/updates_page.py | 40 +++++- tierd-ui/src/i18n/locales/en.ts | 2 + tierd-ui/src/i18n/locales/nl.ts | 2 + tierd-ui/src/pages/Updates/Updates.tsx | 23 +++- tierd/internal/updater/run.go | 51 +++++++- tierd/internal/updater/run_test.go | 170 ++++++++++++++++++++++++- 6 files changed, 279 insertions(+), 9 deletions(-) diff --git a/tests/browser/updates_page.py b/tests/browser/updates_page.py index c6af6f01..45c77ed8 100644 --- a/tests/browser/updates_page.py +++ b/tests/browser/updates_page.py @@ -6,8 +6,10 @@ The page lists available updates, checks for and installs them on request while following the run's log, shows a failed run's errors (an unreachable -archive must not be silent), says whether a restart is needed and why, and -restarts only after the operator confirms. +archive must not be silent), says when a failed install has since been +superseded by successful unattended upgrades and when an install succeeded +although a package source couldn't be refreshed, says whether a restart is +needed and why, and restarts only after the operator confirms. """ import json import os @@ -18,6 +20,7 @@ base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:5173" REASON = "smoothfs 0.3.2 is installed; running 0.3.1" RESOLVE_ERR = "Err:5 https://updates.rakuensoftware.com/public common InRelease: Temporary failure resolving 'updates.rakuensoftware.com'" +BCACHEFS_ERR = "Err:4 https://apt.bcachefs.org/trixie bcachefs-tools-release InRelease: Could not connect to apt.bcachefs.org:443 (5.161.236.196). - connect (111: Connection refused)" LOG = "== smoothnas-update start install 2026-10-29T18:10:00Z\nSetting up smoothfs-dkms (0.3.2-1) ...\n" @@ -34,6 +37,23 @@ def status(scenario, state): "last_run": {"action": "install", "started_at": "t0", "finished_at": "t1", "exit_code": 0, "errors": []}} return {"running": False, "available": [], "reboot": no_reboot, "last_run": {"action": "check", "started_at": "t0", "finished_at": "2026-10-29T18:00:00Z", "exit_code": 0, "errors": []}} + # The WTR Max on 2026-11-02: an install started here failed on Nov 1 + # because apt.bcachefs.org was down; unattended upgrades have succeeded + # since. That failure is history, not an error. + if scenario == "install_failed_superseded": + return {"running": False, "available": [], "reboot": no_reboot, + "last_run": {"action": "install", "started_at": "2026-11-01T15:36:21Z", "finished_at": "2026-11-01T15:36:28Z", + "exit_code": 100, "errors": [BCACHEFS_ERR], "superseded_at": "2026-11-02T13:38:27Z"}} + # Negative control: the same failure, nothing has succeeded since. + if scenario == "install_failed": + return {"running": False, "available": [], "reboot": no_reboot, + "last_run": {"action": "install", "started_at": "2026-11-01T15:36:21Z", "finished_at": "2026-11-01T15:36:28Z", + "exit_code": 100, "errors": [BCACHEFS_ERR]}} + # A source was down, but the install went ahead and succeeded. + if scenario == "installed_refresh_failed": + return {"running": False, "available": [], "reboot": no_reboot, + "last_run": {"action": "install", "started_at": "t0", "finished_at": "2026-11-02T15:00:00Z", + "exit_code": 0, "refresh_failed": True, "errors": [BCACHEFS_ERR]}} if scenario == "check_failed": return {"running": False, "available": [], "reboot": no_reboot, "last_run": {"action": "check", "started_at": "t0", "finished_at": "t1", "exit_code": 100, "errors": [RESOLVE_ERR]}} @@ -50,7 +70,8 @@ def status(scenario, state): if os.environ.get("CHROMIUM_PATH"): options["executable_path"] = os.environ["CHROMIUM_PATH"] browser = playwright.chromium.launch(**options) - for scenario in ("restart_needed", "up_to_date", "check_failed", "available"): + for scenario in ("restart_needed", "up_to_date", "check_failed", "install_failed_superseded", + "install_failed", "installed_refresh_failed", "available"): context = browser.new_context(ignore_https_errors=os.environ.get("ALLOW_INSECURE_TLS") == "1") context.add_init_script("localStorage.setItem('sn_session','true'); localStorage.setItem('sn_user','qa');") reboots = [] @@ -107,6 +128,19 @@ def respond(route): page.get_by_role("button", name="Check for updates", exact=True).click() expect(page.get_by_text("Checking for updates…", exact=True)).to_be_visible() assert page.get_by_text("Installing updates…", exact=False).count() == 0, "a check was labelled as an install" + elif scenario == "install_failed_superseded": + expect(page.get_by_text("SmoothNAS is up to date.", exact=False)).to_be_visible() + expect(page.get_by_text("automatic updates have installed successfully since (2026-11-02T13:38:27Z)", exact=False)).to_be_visible() + assert page.get_by_text("Installing updates failed", exact=False).count() == 0, "a superseded failure is shown as current" + assert page.locator(".error-msg").count() == 0, "a superseded failure is shown as an error" + elif scenario == "install_failed": + expect(page.get_by_text("Installing updates failed (2026-11-01T15:36:28Z).", exact=True)).to_be_visible() + expect(page.get_by_text(BCACHEFS_ERR, exact=True)).to_be_visible() + assert page.get_by_text("automatic updates have installed", exact=False).count() == 0 + elif scenario == "installed_refresh_failed": + expect(page.get_by_text("Updates were installed (2026-11-02T15:00:00Z), but these package sources could not be refreshed:", exact=True)).to_be_visible() + expect(page.get_by_text(BCACHEFS_ERR, exact=True)).to_be_visible() + assert page.get_by_text("Installing updates failed", exact=False).count() == 0, "a successful install is shown as failed" elif scenario == "check_failed": expect(page.get_by_text("Checking for updates failed", exact=False)).to_be_visible() expect(page.get_by_text(RESOLVE_ERR, exact=True)).to_be_visible() diff --git a/tierd-ui/src/i18n/locales/en.ts b/tierd-ui/src/i18n/locales/en.ts index 153133ec..1a6590fb 100644 --- a/tierd-ui/src/i18n/locales/en.ts +++ b/tierd-ui/src/i18n/locales/en.ts @@ -1106,6 +1106,8 @@ const en: LanguageTranslations = { 'updates.label.installing': 'Installing updates… This continues even if you leave this page.', 'updates.label.checkFailed': 'Checking for updates failed ({time}).', 'updates.label.installFailed': 'Installing updates failed ({time}).', + 'updates.label.installFailedSuperseded': 'An update started here failed ({failed}); automatic updates have installed successfully since ({time}).', + 'updates.label.installedRefreshFailed': 'Updates were installed ({time}), but these package sources could not be refreshed:', 'updates.label.upToDate': 'SmoothNAS is up to date.', 'updates.label.lastChecked': '(last checked {time})', 'updates.label.available': '{count} update(s) available', diff --git a/tierd-ui/src/i18n/locales/nl.ts b/tierd-ui/src/i18n/locales/nl.ts index 6d544d17..582b8baf 100644 --- a/tierd-ui/src/i18n/locales/nl.ts +++ b/tierd-ui/src/i18n/locales/nl.ts @@ -1126,6 +1126,8 @@ const nl: LanguageTranslations = { 'updates.label.installing': 'Updates worden geïnstalleerd… Dit gaat door, ook als je deze pagina verlaat.', 'updates.label.checkFailed': 'Zoeken naar updates mislukt ({time}).', 'updates.label.installFailed': 'Installeren van updates mislukt ({time}).', + 'updates.label.installFailedSuperseded': 'Een hier gestarte update is mislukt ({failed}); automatische updates zijn sindsdien wel geslaagd ({time}).', + 'updates.label.installedRefreshFailed': 'Updates zijn geïnstalleerd ({time}), maar deze pakketbronnen konden niet worden vernieuwd:', 'updates.label.upToDate': 'SmoothNAS is up-to-date.', 'updates.label.lastChecked': '(laatst gecontroleerd {time})', 'updates.label.available': '{count} update(s) beschikbaar', diff --git a/tierd-ui/src/pages/Updates/Updates.tsx b/tierd-ui/src/pages/Updates/Updates.tsx index ab94e4df..6670f2db 100644 --- a/tierd-ui/src/pages/Updates/Updates.tsx +++ b/tierd-ui/src/pages/Updates/Updates.tsx @@ -25,6 +25,11 @@ interface UpdateStatus { finished_at?: string; exit_code?: number; errors: string[]; + // An install whose package-list refresh failed for some source; the + // install still ran and exit_code is its result. + refresh_failed?: boolean; + // A failed install that unattended upgrades have since succeeded past. + superseded_at?: string; } | null; reboot: RebootStatus; } @@ -119,7 +124,9 @@ export default function Updates() { const last = status?.last_run; const runningAction = (last?.finished_at ? null : last?.action) || startedAction; - const lastFailed = !!last && !status?.running && last.exit_code !== undefined && last.exit_code !== 0; + const lastFailed = !!last && !status?.running && last.exit_code !== undefined && last.exit_code !== 0 && !last.superseded_at; + const refreshFailed = !!last && !status?.running && last.exit_code === 0 && !!last.refresh_failed; + const superseded = !!last && !status?.running && !!last.superseded_at; const available = status?.available ?? []; return ( @@ -150,12 +157,24 @@ export default function Updates() { )} + ) : refreshFailed ? ( +
+ {t('updates.label.installedRefreshFailed', { time: last!.finished_at || last!.started_at })} + +
) : available.length === 0 ? (

{t('updates.label.upToDate')} - {last?.finished_at && {t('updates.label.lastChecked', { time: last.finished_at })}} + {last?.finished_at && !superseded && {t('updates.label.lastChecked', { time: last.finished_at })}}

) : null} + {superseded && ( +

+ {t('updates.label.installFailedSuperseded', { failed: last!.finished_at || last!.started_at, time: last!.superseded_at! })} +

+ )} {available.length > 0 && !status.running && (
diff --git a/tierd/internal/updater/run.go b/tierd/internal/updater/run.go index f4358a7f..3a9ef82c 100644 --- a/tierd/internal/updater/run.go +++ b/tierd/internal/updater/run.go @@ -9,6 +9,7 @@ import ( "regexp" "strconv" "strings" + "time" ) // Operator-started update runs. unattended-upgrades keeps the NAS current on @@ -25,6 +26,10 @@ const runUnit = "smoothnas-update" var ( runLogPath = "/var/log/smoothnas/update.log" + // apt.systemd.daily touches this only when unattended-upgrade exits 0, so + // its time is when updates last installed without an error. + upgradeStampPath = "/var/lib/apt/periodic/upgrade-stamp" + // systemctl and systemd-run are vars so tests can fake them. unitActive = func() bool { return exec.Command("systemctl", "is-active", "--quiet", runUnit).Run() == nil @@ -61,6 +66,14 @@ const aptRepair = "{ dpkg --configure -a || true; } && " + "apt-get -o DPkg::Lock::Timeout=600 -o Dpkg::Options::=--force-confdef " + "-o Dpkg::Options::=--force-confold -y --fix-broken install" +// installCmd refreshes the package lists, then installs whatever they offer. +// A source that can't be refreshed (a third-party archive that is down) must +// not stop SmoothNAS's own updates from installing: the refresh result is +// recorded on its own line and reported, and the install still runs, from +// the lists as of the last successful refresh for that source. +const installCmd = aptUpdate + "; refresh=$?; echo \"" + refreshMarker + " $refresh\"; " + + aptRepair + " && " + aptUpgrade + // Run actions. const ( ActionCheck = "check" @@ -80,7 +93,7 @@ func Start(action string) error { case ActionCheck: script = runScript(ActionCheck, aptUpdate, runLogPath) case ActionInstall: - script = runScript(ActionInstall, aptUpdate+" && "+aptRepair+" && "+aptUpgrade, runLogPath) + script = runScript(ActionInstall, installCmd, runLogPath) default: return fmt.Errorf("unknown update action %q", action) } @@ -102,8 +115,9 @@ func runScript(action, cmd, logPath string) string { } const ( - startMarker = "== smoothnas-update start" - endMarker = "== smoothnas-update exit" + startMarker = "== smoothnas-update start" + endMarker = "== smoothnas-update exit" + refreshMarker = "== smoothnas-update refresh-exit" ) // Package is an update available for an installed package. @@ -120,6 +134,13 @@ type LastRun struct { FinishedAt string `json:"finished_at,omitempty"` ExitCode *int `json:"exit_code,omitempty"` Errors []string `json:"errors"` + // RefreshFailed: an install whose package-list refresh failed for some + // source; the install itself still ran, and ExitCode is its result. + RefreshFailed bool `json:"refresh_failed,omitempty"` + // SupersededAt: this failed install is older than a later unattended + // run that installed updates without an error, at this time. The + // failure is history, not the NAS's current state. + SupersededAt string `json:"superseded_at,omitempty"` } // UpdateStatus is what the Updates page shows. @@ -141,9 +162,29 @@ func GetUpdateStatus() UpdateStatus { st.LastRun = parseRunLog(f) f.Close() } + markSuperseded(st.LastRun) return st } +// markSuperseded sets SupersededAt on a failed install that unattended +// upgrades have since succeeded past. A failed check is left alone: the +// unattended refresh runs in apt's default mode, which exits 0 even when an +// archive is unreachable, so it can't show that a check would now pass. +func markSuperseded(run *LastRun) { + if run == nil || run.Action != ActionInstall || run.ExitCode == nil || *run.ExitCode == 0 { + return + } + failed, err := time.Parse(time.RFC3339, run.FinishedAt) + if err != nil { + return + } + info, err := os.Stat(upgradeStampPath) + if err != nil || !info.ModTime().After(failed) { + return + } + run.SupersededAt = info.ModTime().UTC().Format(time.RFC3339) +} + // ReadLog returns the run log from byte offset on, and the offset to read // from next. A run truncates the log, so an offset past the end restarts at 0. func ReadLog(offset int64) (string, int64, error) { @@ -211,6 +252,10 @@ func parseRunLog(r io.Reader) *LastRun { if len(f) > 1 { run.StartedAt = f[1] } + case run != nil && strings.HasPrefix(line, refreshMarker+" "): + if code, err := strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, refreshMarker+" "))); err == nil { + run.RefreshFailed = code != 0 + } case run != nil && strings.HasPrefix(line, endMarker+" "): f := strings.Fields(strings.TrimPrefix(line, endMarker+" ")) if len(f) > 0 { diff --git a/tierd/internal/updater/run_test.go b/tierd/internal/updater/run_test.go index 5cd04fe3..75844176 100644 --- a/tierd/internal/updater/run_test.go +++ b/tierd/internal/updater/run_test.go @@ -1,10 +1,13 @@ package updater import ( + "fmt" "os" + "os/exec" "path/filepath" "strings" "testing" + "time" ) func TestParseUpgradable(t *testing.T) { @@ -84,7 +87,7 @@ func TestStart(t *testing.T) { if data, _ := os.ReadFile(runLogPath); len(data) != 0 { t.Errorf("expected the log truncated for a new run, got %q", data) } - for _, want := range []string{"APT::Update::Error-Mode=any", "dpkg --configure -a", "--fix-broken install", "full-upgrade", "--force-confold", startMarker + " install", endMarker, runLogPath} { + for _, want := range []string{"APT::Update::Error-Mode=any", refreshMarker, "dpkg --configure -a", "--fix-broken install", "full-upgrade", "--force-confold", startMarker + " install", endMarker, runLogPath} { if !strings.Contains(script, want) { t.Errorf("install script missing %q: %s", want, script) } @@ -121,3 +124,168 @@ func TestReadLog(t *testing.T) { t.Fatalf("ReadLog past end = %q %d", text, next) } } + +// The WTR Max's Updates page on 2026-11-02: an install started from the page +// on Nov 1 failed because apt.bcachefs.org refused connections, and the +// page still said "Installing updates failed" a day later, after unattended +// upgrades had installed kernel 7.2.15-84 and OpenZFS without an error. +const wtrMaxFailedInstall = `== smoothnas-update start install 2026-11-01T15:36:21Z +Err:4 https://apt.bcachefs.org/trixie bcachefs-tools-release InRelease + Could not connect to apt.bcachefs.org:443 (5.161.236.196). - connect (111: Connection refused) +E: Failed to fetch https://apt.bcachefs.org/trixie/dists/bcachefs-tools-release/InRelease Cannot initiate the connection to apt.bcachefs.org:443 (2a01:4ff:f0:49f3::1). - connect (101: Network is unreachable) +E: Some index files failed to download. They have been ignored, or old ones used instead. +== smoothnas-update exit 100 2026-11-01T15:36:28Z +` + +func writeStamp(t *testing.T, at time.Time) { + t.Helper() + upgradeStampPath = filepath.Join(t.TempDir(), "upgrade-stamp") + if err := os.WriteFile(upgradeStampPath, nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(upgradeStampPath, at, at); err != nil { + t.Fatal(err) + } +} + +func TestMarkSupersededAfterUnattendedSuccess(t *testing.T) { + writeStamp(t, time.Date(2026, 11, 2, 13, 38, 27, 0, time.UTC)) + run := parseRunLog(strings.NewReader(wtrMaxFailedInstall)) + markSuperseded(run) + if run.SupersededAt != "2026-11-02T13:38:27Z" { + t.Fatalf("failed install older than a successful unattended upgrade: SupersededAt = %q", run.SupersededAt) + } + if run.ExitCode == nil || *run.ExitCode != 100 || len(run.Errors) == 0 { + t.Fatalf("superseding must keep the failure's record: %+v", run) + } +} + +func TestMarkSupersededLeavesCurrentFailures(t *testing.T) { + failedAt := time.Date(2026, 11, 1, 15, 36, 28, 0, time.UTC) + cases := []struct { + name string + log string + stamp *time.Time + }{ + {"unattended last succeeded before the failure", wtrMaxFailedInstall, ptr(failedAt.Add(-time.Hour))}, + {"unattended succeeded in the same second", wtrMaxFailedInstall, ptr(failedAt)}, + {"no unattended success recorded", wtrMaxFailedInstall, nil}, + {"failed check", strings.Replace(wtrMaxFailedInstall, "start install", "start check", 1), ptr(failedAt.Add(time.Hour))}, + {"successful install", strings.Replace(wtrMaxFailedInstall, "exit 100", "exit 0", 1), ptr(failedAt.Add(time.Hour))}, + {"unfinished install", "== smoothnas-update start install 2026-11-01T15:36:21Z\n", ptr(failedAt.Add(time.Hour))}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if c.stamp != nil { + writeStamp(t, *c.stamp) + } else { + upgradeStampPath = filepath.Join(t.TempDir(), "absent") + } + run := parseRunLog(strings.NewReader(c.log)) + markSuperseded(run) + if run.SupersededAt != "" { + t.Fatalf("SupersededAt = %q, want empty", run.SupersededAt) + } + }) + } + markSuperseded(nil) // no run yet: must not panic +} + +func ptr(t time.Time) *time.Time { return &t } + +func TestGetUpdateStatusReportsSuperseded(t *testing.T) { + runLogPath = filepath.Join(t.TempDir(), "update.log") + if err := os.WriteFile(runLogPath, []byte(wtrMaxFailedInstall), 0o644); err != nil { + t.Fatal(err) + } + writeStamp(t, time.Date(2026, 11, 2, 13, 38, 27, 0, time.UTC)) + unitActive = func() bool { return false } + aptListUpgradable = func() ([]byte, error) { return []byte("Listing... Done\n"), nil } + st := GetUpdateStatus() + if st.LastRun == nil || st.LastRun.SupersededAt == "" { + t.Fatalf("status must report the failure as superseded: %+v", st.LastRun) + } +} + +// Run the install script Start builds, with stub apt-get and dpkg, and read +// its log back the way the Updates page does. +func runInstallScript(t *testing.T, updateExit, upgradeExit int) (*LastRun, string) { + t.Helper() + dir := t.TempDir() + calls := filepath.Join(dir, "calls") + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + stub := fmt.Sprintf(`#!/bin/sh +echo "$(basename "$0") $*" >> %[1]q +case "$*" in + *" update") [ %[2]d -eq 0 ] || echo "E: Some index files failed to download."; exit %[2]d ;; + *full-upgrade*) exit %[3]d ;; +esac +exit 0 +`, calls, updateExit, upgradeExit) + for _, name := range []string{"apt-get", "dpkg"} { + if err := os.WriteFile(filepath.Join(bin, name), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + } + runLogPath = filepath.Join(dir, "update.log") + unitActive = func() bool { return false } + var script string + startUnit = func(s string) error { script = s; return nil } + if err := Start(ActionInstall); err != nil { + t.Fatal(err) + } + cmd := exec.Command("/bin/sh", "-c", script) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("script: %v: %s", err, out) + } + f, err := os.Open(runLogPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + got, _ := os.ReadFile(calls) + return parseRunLog(f), string(got) +} + +// A source that can't be refreshed must not stop the install. +func TestInstallRunsWhenRefreshFails(t *testing.T) { + run, calls := runInstallScript(t, 100, 0) + if !strings.Contains(calls, "full-upgrade") { + t.Fatalf("install did not run after the refresh failed; calls:\n%s", calls) + } + if run.ExitCode == nil || *run.ExitCode != 0 { + t.Fatalf("install succeeded, exit = %v", run.ExitCode) + } + if !run.RefreshFailed { + t.Fatal("refresh failure not recorded") + } + if len(run.Errors) == 0 { + t.Fatal("refresh errors not reported") + } +} + +func TestInstallFailureStillReported(t *testing.T) { + run, _ := runInstallScript(t, 0, 100) + if run.ExitCode == nil || *run.ExitCode != 100 { + t.Fatalf("failed install must report its exit code, got %v", run.ExitCode) + } + if run.RefreshFailed { + t.Fatal("refresh succeeded but was reported as failed") + } +} + +func TestInstallCleanRun(t *testing.T) { + run, calls := runInstallScript(t, 0, 0) + if run.ExitCode == nil || *run.ExitCode != 0 || run.RefreshFailed || len(run.Errors) != 0 { + t.Fatalf("clean run reported as %+v", run) + } + for _, want := range []string{"apt-get -o APT::Update::Error-Mode=any", "dpkg --configure -a", "--fix-broken install", "full-upgrade"} { + if !strings.Contains(calls, want) { + t.Errorf("install did not run %q; calls:\n%s", want, calls) + } + } +}