From 8ae93e0faa830ea0c2c7830e4e1d7bd9209f10c1 Mon Sep 17 00:00:00 2001 From: z2z23n0 Date: Sat, 18 Jul 2026 11:30:18 +0800 Subject: [PATCH 1/5] fix: surface scheduled update failures --- README.md | 2 + internal/cli/app.go | 1 + internal/cli/cli_test.go | 46 +++- internal/cli/read_commands.go | 35 +-- internal/cli/worker_commands.go | 70 +++++- internal/config/config_test.go | 3 + internal/config/paths.go | 7 +- internal/doctor/doctor.go | 220 ++++++++++++++++-- internal/doctor/doctor_test.go | 51 +++- internal/model/types.go | 11 + internal/notify/desktop.go | 47 ++++ internal/notify/desktop_test.go | 33 +++ internal/reconcile/types.go | 31 +-- internal/reconcile/worker.go | 132 ++++++++++- internal/reconcile/worker_test.go | 67 ++++++ internal/scheduler/scheduler.go | 130 +++++++++-- internal/scheduler/scheduler_test.go | 33 ++- internal/store/bundles_test.go | 8 +- internal/store/lifecycle.go | 12 +- .../store/migrations/0006_reconcile_runs.sql | 13 ++ internal/store/reconcile_runs.go | 64 +++++ internal/store/store.go | 2 +- internal/store/store_test.go | 28 ++- internal/watchdog/watchdog.go | 113 +++++++++ internal/watchdog/watchdog_test.go | 89 +++++++ 25 files changed, 1152 insertions(+), 96 deletions(-) create mode 100644 internal/notify/desktop.go create mode 100644 internal/notify/desktop_test.go create mode 100644 internal/store/migrations/0006_reconcile_runs.sql create mode 100644 internal/store/reconcile_runs.go create mode 100644 internal/watchdog/watchdog.go create mode 100644 internal/watchdog/watchdog_test.go diff --git a/README.md b/README.md index bbd561c..04abef8 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ SessionStart / ToolUse / 每日任务 / 用户命令 - Hook 热路径不联网、不合并、不调用模型，SQLite 使用 `busy_timeout=0`；数据库繁忙或输入异常时 fail-open。 - `kick` 只启动一个脱离当前会话的一次性 worker。全局文件锁保证并发 Session 不会并行更新。 - macOS 使用 launchd，Linux 使用 systemd user timer；两者每天启动一次 `reconcile --once`，没有常驻 ToolTend 进程。 +- 每轮 reconcile 都会持久化完整运行状态；主任务之后由独立 watchdog 检查漏跑、失败或未完成状态。失败默认发送桌面通知，并在下次 Codex/Claude SessionStart 时补充提醒。 +- macOS 调度输出保存在 `~/.local/state/tooltend/logs/`，不会再丢弃到 `/dev/null`；`tooltend status` 和 `tooltend doctor` 会显示最近一次完整 reconcile 的结果。 - 未执行 `bundles configure` 的 Bundle 不检查更新、不下载，也不调用安装器。 - Bundle 更新先完成所有 Artifact 的解析、校验和 staging，再按物理 Installation 激活；失败时按相反顺序补偿。 - Bundle 事务使用步骤 journal。中断、失败、回滚和健康检查都有 Bundle 级 Receipt 可审计。 diff --git a/internal/cli/app.go b/internal/cli/app.go index 88f1567..988b7fe 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -137,6 +137,7 @@ func New(options Options) *cobra.Command { a.newHookCommand(), a.newKickCommand(), a.newReconcileCommand(), + a.newWatchdogCommand(), a.newVersionCommand(), ) root.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error { diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index fd81f11..837677e 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -22,6 +22,7 @@ import ( "github.com/z2z23n0/tooltend/internal/inventory" "github.com/z2z23n0/tooltend/internal/lockfile" "github.com/z2z23n0/tooltend/internal/model" + "github.com/z2z23n0/tooltend/internal/reconcile" "github.com/z2z23n0/tooltend/internal/store" ) @@ -31,7 +32,7 @@ func TestCommandTreeContainsCompleteV1Surface(t *testing.T) { "init", "scan", "status", "components list", "components show", "policy set", "bundles list", "bundles show", "bundles configure", "bundles update", "bundles rollback", "bundles history", "bundles doctor", "update", "review", "history", "rollback", "adopt", "project init", "project export", - "project sync", "self status", "self update", "doctor", "hook", "kick", "reconcile", "version", + "project sync", "self status", "self update", "doctor", "hook", "kick", "reconcile", "watchdog", "version", } { if _, _, err := command.Find(strings.Fields(path)); err != nil { t.Fatalf("missing command %q: %v", path, err) @@ -147,7 +148,7 @@ func TestResetStateRestoresOldStateWhenSchedulerReactivationFails(t *testing.T) } configHash := fileHashOrEmpty(paths.ConfigFile) databaseHash := fileHashOrEmpty(paths.DatabaseFile) - runner.failAt = runner.calls + 3 // deactivate and best-effort bootout succeed; final registration fails once + runner.failAt = runner.calls + 4 // two deactivations and best-effort bootout succeed; final registration fails once out.Reset() command = New(options) command.SetArgs([]string{"init", "--reset-state", "--yes", "--json"}) @@ -322,6 +323,47 @@ type successfulRunner struct { failAt int } +type notificationRunner struct { + name string + args []string +} + +func (r *notificationRunner) Run(_ context.Context, name string, args ...string) (execx.Result, error) { + r.name, r.args = name, append([]string(nil), args...) + return execx.Result{}, nil +} + +func TestScheduledFailureSendsDesktopNotification(t *testing.T) { + home := t.TempDir() + paths := config.ResolveWith(home, func(name string) string { + if name == config.EnvHome { + return filepath.Join(home, "tooltend") + } + return "" + }) + if err := paths.Ensure(); err != nil { + t.Fatal(err) + } + if err := config.SaveAtomic(paths.ConfigFile, config.Default()); err != nil { + t.Fatal(err) + } + runner := ¬ificationRunner{} + a := newApp(Options{HomeDir: home, Runner: runner, Getenv: func(name string) string { + if name == config.EnvHome { + return filepath.Join(home, "tooltend") + } + return "" + }}) + a.notifyScheduledOutcome(context.Background(), paths, reconcile.RunResult{Failed: 2, FailureNotificationQueued: true}, nil) + wantName := "notify-send" + if runtime.GOOS == "darwin" { + wantName = "/usr/bin/osascript" + } + if runner.name != wantName || !strings.Contains(strings.Join(runner.args, " "), "2 task(s) failed") { + t.Fatalf("notification call = %s %#v", runner.name, runner.args) + } +} + func (r *successfulRunner) Run(context.Context, string, ...string) (execx.Result, error) { r.calls++ if r.failAt > 0 && r.calls == r.failAt { diff --git a/internal/cli/read_commands.go b/internal/cli/read_commands.go index cc01024..97bc4e0 100644 --- a/internal/cli/read_commands.go +++ b/internal/cli/read_commands.go @@ -17,19 +17,20 @@ import ( ) type statusData struct { - Initialized bool `json:"initialized"` - Issues []string `json:"issues,omitempty"` - Bundles int `json:"bundles"` - ConfiguredBundles int `json:"configured_bundles"` - ManagedBundles int `json:"managed_bundles"` - ObservedBundles int `json:"observed_bundles"` - UnconfiguredBundles int `json:"unconfigured_bundles"` - UnresolvedBundles int `json:"unresolved_bundles"` - UpdatesAvailable int `json:"updates_available"` - FailedTransactions int `json:"failed_transactions"` - PendingTasks int `json:"pending_tasks"` - UnfinishedActions int `json:"unfinished_transactions"` - Debug statusDebug `json:"debug"` + Initialized bool `json:"initialized"` + Issues []string `json:"issues,omitempty"` + Bundles int `json:"bundles"` + ConfiguredBundles int `json:"configured_bundles"` + ManagedBundles int `json:"managed_bundles"` + ObservedBundles int `json:"observed_bundles"` + UnconfiguredBundles int `json:"unconfigured_bundles"` + UnresolvedBundles int `json:"unresolved_bundles"` + UpdatesAvailable int `json:"updates_available"` + FailedTransactions int `json:"failed_transactions"` + PendingTasks int `json:"pending_tasks"` + UnfinishedActions int `json:"unfinished_transactions"` + LatestReconcile *model.ReconcileRun `json:"latest_reconcile,omitempty"` + Debug statusDebug `json:"debug"` } type statusDebug struct { @@ -101,6 +102,14 @@ func (a *App) newStatusCommand() *cobra.Command { return statusData{Initialized: false, Issues: []string{"project_inventory_missing"}}, nil } result := statusData{Initialized: true} + if latest, latestErr := database.LatestReconcileRun(ctx); latestErr == nil { + result.LatestReconcile = &latest + if latest.Status == "failed" { + result.Issues = append(result.Issues, "latest_reconcile_failed") + } + } else if !errors.Is(latestErr, sql.ErrNoRows) { + return nil, latestErr + } components, err := database.ListComponents(ctx) if err != nil { return nil, err diff --git a/internal/cli/worker_commands.go b/internal/cli/worker_commands.go index 6520fe8..95aeb4e 100644 --- a/internal/cli/worker_commands.go +++ b/internal/cli/worker_commands.go @@ -22,9 +22,11 @@ import ( "github.com/z2z23n0/tooltend/internal/kick" "github.com/z2z23n0/tooltend/internal/lifecycle" "github.com/z2z23n0/tooltend/internal/model" + "github.com/z2z23n0/tooltend/internal/notify" "github.com/z2z23n0/tooltend/internal/objectstore" "github.com/z2z23n0/tooltend/internal/reconcile" "github.com/z2z23n0/tooltend/internal/store" + "github.com/z2z23n0/tooltend/internal/watchdog" ) type hookStore struct{ database *store.Store } @@ -54,6 +56,10 @@ func (s hookStore) TakePending(ctx context.Context, limit int) ([]string, error) } result := make([]string, 0, len(values)) for _, value := range values { + if strings.TrimSpace(value.Message) != "" { + result = append(result, value.Message) + continue + } hash := value.CandidateHash if len(hash) > 12 { hash = hash[:12] @@ -117,7 +123,7 @@ func (a *App) reconcileDue(ctx context.Context, paths config.Paths, database *st interval = cfg.Check.Interval } var latest sql.NullString - if err := database.DB().QueryRowContext(ctx, `SELECT max(finished_at) FROM scans WHERE status='succeeded'`).Scan(&latest); err != nil { + if err := database.DB().QueryRowContext(ctx, `SELECT max(finished_at) FROM reconcile_runs WHERE status='succeeded'`).Scan(&latest); err != nil { return false } if !latest.Valid { @@ -168,11 +174,71 @@ func (a *App) newReconcileCommand() *cobra.Command { if a.global.DryRun { return map[string]any{"dry_run": true, "reason": reason, "state_dir": paths.StateDir}, nil } - return a.reconcileOnce(ctx, paths, reason) + value, runErr := a.reconcileOnce(ctx, paths, reason) + if reason == reconcile.ReasonScheduled { + a.notifyScheduledOutcome(ctx, paths, value, runErr) + } + return value, runErr + }) + return command +} + +func (a *App) newWatchdogCommand() *cobra.Command { + var maxAge time.Duration + command := &cobra.Command{Use: "watchdog", Short: "Alert when scheduled reconciliation does not complete", Hidden: true, Args: cobra.NoArgs} + command.Flags().DurationVar(&maxAge, "max-age", 2*time.Hour, "maximum age of the latest successful reconciliation") + command.RunE = a.run("watchdog", func(ctx context.Context) (any, error) { + if maxAge <= 0 { + return nil, cliError("invalid_argument", "watchdog max age must be positive", nil) + } + paths, err := a.paths() + if err != nil { + return nil, err + } + if a.global.DryRun { + return map[string]any{"dry_run": true, "max_age": maxAge.String(), "state_dir": paths.StateDir}, nil + } + desktop := notify.Desktop{Runner: a.runner} + cfg, err := config.Load(paths.ConfigFile) + if err != nil { + _ = desktop.Send(ctx, "ToolTend", "Scheduled update state cannot be checked. Run `tooltend doctor` for details.") + return nil, err + } + database, err := store.OpenRW(paths.DatabaseFile) + if err != nil { + _ = desktop.Send(ctx, "ToolTend", "Scheduled update state cannot be opened. Run `tooltend doctor` for details.") + return nil, err + } + defer database.Close() + return (watchdog.Service{ + Database: database, + Notifier: desktop, + Enabled: cfg.Notify.Mode != model.NotifyNone, + }).Check(ctx, maxAge) }) return command } +func (a *App) notifyScheduledOutcome(ctx context.Context, paths config.Paths, value any, runErr error) { + cfg, cfgErr := config.Load(paths.ConfigFile) + if cfgErr == nil && cfg.Notify.Mode == model.NotifyNone { + return + } + message := "" + result, _ := value.(reconcile.RunResult) + switch { + case runErr != nil && (result.RunID == "" || result.FailureNotificationQueued || result.Failed > 0): + message = "Scheduled update failed before completion. Run `tooltend doctor` for details." + case result.Failed > 0 && result.FailureNotificationQueued: + message = fmt.Sprintf("Scheduled update failed: %d task(s) failed. Run `tooltend doctor` for details.", result.Failed) + case cfgErr == nil && cfg.Notify.Mode == model.NotifyAll && result.Succeeded > 0: + message = fmt.Sprintf("Scheduled update completed: %d task(s) succeeded.", result.Succeeded) + } + if message != "" { + _ = (notify.Desktop{Runner: a.runner}).Send(ctx, "ToolTend", message) + } +} + func (a *App) reconcileOnce(ctx context.Context, paths config.Paths, reason string) (any, error) { cfg, err := config.Load(paths.ConfigFile) if err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9aec416..da563a3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -32,6 +32,9 @@ func TestResolveWithUsesXDGAndIgnoresRelativeXDG(t *testing.T) { if p.DatabaseFile != filepath.Join(home, ".local", "state", "tooltend", "state.db") { t.Fatalf("state path = %s", p.DatabaseFile) } + if p.LogsDir != filepath.Join(home, ".local", "state", "tooltend", "logs") { + t.Fatalf("logs path = %s", p.LogsDir) + } } func TestSaveAtomicRoundTrip(t *testing.T) { diff --git a/internal/config/paths.go b/internal/config/paths.go index e0053b0..5e7d056 100644 --- a/internal/config/paths.go +++ b/internal/config/paths.go @@ -13,6 +13,7 @@ type Paths struct { ConfigDir string `json:"config_dir"` ConfigFile string `json:"config_file"` StateDir string `json:"state_dir"` + LogsDir string `json:"logs_dir"` DatabaseFile string `json:"database_file"` DataDir string `json:"data_dir"` ObjectsDir string `json:"objects_dir"` @@ -54,6 +55,7 @@ func pathsFor(configDir, stateDir, dataDir, shimDir string) Paths { ConfigDir: configDir, ConfigFile: filepath.Join(configDir, "config.toml"), StateDir: stateDir, + LogsDir: filepath.Join(stateDir, "logs"), DatabaseFile: filepath.Join(stateDir, "state.db"), DataDir: dataDir, ObjectsDir: filepath.Join(dataDir, "objects"), @@ -86,7 +88,10 @@ func cleanRoot(value, home string) string { // Ensure creates ToolTend-owned roots. Call it only after a confirmed write plan. func (p Paths) Ensure() error { - for _, dir := range []string{p.ConfigDir, p.StateDir, p.ObjectsDir, p.StagingDir, p.GenerationsDir, p.RuntimesDir} { + for _, dir := range []string{p.ConfigDir, p.StateDir, p.LogsDir, p.ObjectsDir, p.StagingDir, p.GenerationsDir, p.RuntimesDir} { + if strings.TrimSpace(dir) == "" { + continue + } if err := os.MkdirAll(dir, 0o700); err != nil { return fmt.Errorf("config: create %s: %w", dir, err) } diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index c71094c..28abfe7 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -5,12 +5,16 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "encoding/xml" "errors" "fmt" "os" "path/filepath" "runtime" + "slices" + "strconv" "strings" + "time" "github.com/z2z23n0/tooltend/internal/config" "github.com/z2z23n0/tooltend/internal/execx" @@ -54,6 +58,7 @@ type Options struct { func Run(ctx context.Context, paths config.Paths) Report { report := Report{Healthy: true, Checks: []Check{}} + checkInterval := 24 * time.Hour appendCheck := func(check Check) { report.Checks = append(report.Checks, check) if check.Level == LevelError { @@ -68,6 +73,7 @@ func Run(ctx context.Context, paths config.Paths) Report { } appendCheck(Check{Name: "config", Level: level, Message: safeMessage("configuration unavailable", err), Repairable: true}) } else { + checkInterval = value.Check.Interval appendCheck(Check{Name: "config", Level: LevelOK, Message: fmt.Sprintf("configuration version %d is valid", value.Version)}) } @@ -121,10 +127,11 @@ func Run(ctx context.Context, paths config.Paths) Report { default: appendCheck(Check{Name: "bundle_coverage", Level: LevelOK, Message: fmt.Sprintf("%d of %d bundles are configured", counts.Configured, counts.Total)}) } + appendCheck(checkReconcileRun(ctx, database, checkInterval, time.Now())) } } - for name, path := range map[string]string{"objects": paths.ObjectsDir, "staging": paths.StagingDir, "generations": paths.GenerationsDir} { + for name, path := range map[string]string{"objects": paths.ObjectsDir, "staging": paths.StagingDir, "generations": paths.GenerationsDir, "logs": paths.LogsDir} { info, err := os.Stat(path) switch { case errors.Is(err, os.ErrNotExist): @@ -204,14 +211,14 @@ func checkScheduler(paths config.Paths, home string) Check { for _, schedulerPath := range files { info, err := os.Stat(schedulerPath) if err != nil || !info.Mode().IsRegular() { - return Check{Name: "scheduler", Level: LevelWarning, Message: "daily one-shot schedule is not installed", Repairable: true} + return Check{Name: "scheduler", Level: LevelWarning, Message: "daily schedule is incomplete or not installed", Repairable: true} } if info.Mode().Perm()&0o077 != 0 { return Check{Name: "scheduler", Level: LevelWarning, Message: "daily one-shot schedule permissions are too broad", Repairable: true} } } if len(files) == 0 { - return Check{Name: "scheduler", Level: LevelWarning, Message: "daily one-shot schedule is not installed", Repairable: true} + return Check{Name: "scheduler", Level: LevelWarning, Message: "daily schedule is incomplete or not installed", Repairable: true} } return Check{Name: "scheduler", Level: LevelOK, Message: "daily one-shot schedule is installed"} } @@ -227,18 +234,24 @@ func checkSchedulerWithRunner(ctx context.Context, paths config.Paths, home, exe if runner == nil { runner = execx.ExecRunner{} } - var err error switch runtime.GOOS { case "darwin": - _, err = runner.Run(ctx, "launchctl", "print", fmt.Sprintf("gui/%d/io.tooltend.reconcile", os.Getuid())) + for _, label := range []string{"io.tooltend.reconcile", "io.tooltend.watchdog"} { + result, err := runner.Run(ctx, "launchctl", "print", fmt.Sprintf("gui/%d/%s", os.Getuid(), label)) + if err != nil { + return Check{Name: "scheduler", Level: LevelWarning, Message: "daily schedule is not fully registered", Repairable: true} + } + if code, ok := launchdLastExitCode(string(result.Stdout)); ok && code != 0 { + return Check{Name: "scheduler", Level: LevelError, Message: fmt.Sprintf("%s last exited with code %d", label, code), Repairable: true} + } + } case "linux": - _, err = runner.Run(ctx, "systemctl", "--user", "is-enabled", "tooltend-reconcile.timer") + if _, err := runner.Run(ctx, "systemctl", "--user", "is-enabled", "tooltend-reconcile.timer", "tooltend-watchdog.timer"); err != nil { + return Check{Name: "scheduler", Level: LevelWarning, Message: "daily schedule is not fully registered", Repairable: true} + } default: return check } - if err != nil { - return Check{Name: "scheduler", Level: LevelWarning, Message: "daily one-shot schedule is not registered", Repairable: true} - } return check } @@ -261,38 +274,205 @@ func schedulerFilesMatch(paths config.Paths, home, executable string) bool { } func schedulerFileContentMatches(name, content, executable, stateDir string) bool { - containsAll := func(values ...string) bool { - for _, value := range values { - if !strings.Contains(content, value) { - return false + quoted := func(value string) string { return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(value) + `"` } + line := func(expected string) bool { + for _, value := range strings.Split(content, "\n") { + if strings.TrimSpace(value) == expected { + return true } } - return true + return false } switch name { case "io.tooltend.reconcile.plist": - return containsAll("reconcile", "--once", "--state-dir", filepath.Base(executable), filepath.Base(stateDir), "PATH") + args, ok := plistArray(content, "ProgramArguments") + stdout, stdoutOK := plistString(content, "StandardOutPath") + stderr, stderrOK := plistString(content, "StandardErrorPath") + _, pathOK := plistString(content, "PATH") + return ok && pathOK && stdoutOK && stderrOK && slices.Equal(args, []string{executable, "reconcile", "--once", "--state-dir", stateDir, "--json"}) && + stdout == filepath.Join(stateDir, "logs", "reconcile.stdout.log") && stderr == filepath.Join(stateDir, "logs", "reconcile.stderr.log") + case "io.tooltend.watchdog.plist": + args, ok := plistArray(content, "ProgramArguments") + stdout, stdoutOK := plistString(content, "StandardOutPath") + stderr, stderrOK := plistString(content, "StandardErrorPath") + _, pathOK := plistString(content, "PATH") + return ok && pathOK && stdoutOK && stderrOK && slices.Equal(args, []string{executable, "watchdog", "--max-age", "2h", "--state-dir", stateDir, "--json"}) && + stdout == filepath.Join(stateDir, "logs", "watchdog.stdout.log") && stderr == filepath.Join(stateDir, "logs", "watchdog.stderr.log") case "tooltend-reconcile.service": - return containsAll("reconcile", "--once", "--state-dir", filepath.Base(executable), filepath.Base(stateDir), `Environment="PATH=`) + return strings.Contains(content, `Environment="PATH=`) && line("ExecStart="+quoted(executable)+" reconcile --once --state-dir "+quoted(stateDir)+" --json") + case "tooltend-watchdog.service": + return strings.Contains(content, `Environment="PATH=`) && line("ExecStart="+quoted(executable)+" watchdog --max-age 3h --state-dir "+quoted(stateDir)+" --json") case "tooltend-reconcile.timer": - return containsAll("[Timer]", "OnCalendar=*-*-* ", "RandomizedDelaySec=1h", "Persistent=true", "[Install]", "WantedBy=timers.target") + return timerFileContentMatches(content) + case "tooltend-watchdog.timer": + return timerFileContentMatches(content) default: return false } } +func timerFileContentMatches(content string) bool { + for _, value := range []string{"[Timer]", "OnCalendar=*-*-* ", "RandomizedDelaySec=1h", "Persistent=true", "[Install]", "WantedBy=timers.target"} { + if !strings.Contains(content, value) { + return false + } + } + return true +} + +func plistArray(content, target string) ([]string, bool) { + decoder := xml.NewDecoder(strings.NewReader(content)) + for { + token, err := decoder.Token() + if err != nil { + return nil, false + } + start, ok := token.(xml.StartElement) + if !ok || start.Name.Local != "key" { + continue + } + var key string + if err := decoder.DecodeElement(&key, &start); err != nil || key != target { + continue + } + for { + token, err = decoder.Token() + if err != nil { + return nil, false + } + array, ok := token.(xml.StartElement) + if !ok { + continue + } + if array.Name.Local != "array" { + return nil, false + } + var values []string + for { + token, err = decoder.Token() + if err != nil { + return nil, false + } + switch value := token.(type) { + case xml.StartElement: + if value.Name.Local != "string" { + return nil, false + } + var text string + if err := decoder.DecodeElement(&text, &value); err != nil { + return nil, false + } + values = append(values, text) + case xml.EndElement: + if value.Name.Local == "array" { + return values, true + } + } + } + } + } +} + +func plistString(content, target string) (string, bool) { + decoder := xml.NewDecoder(strings.NewReader(content)) + for { + token, err := decoder.Token() + if err != nil { + return "", false + } + start, ok := token.(xml.StartElement) + if !ok || start.Name.Local != "key" { + continue + } + var key string + if err := decoder.DecodeElement(&key, &start); err != nil || key != target { + continue + } + for { + token, err = decoder.Token() + if err != nil { + return "", false + } + value, ok := token.(xml.StartElement) + if !ok { + continue + } + if value.Name.Local != "string" { + return "", false + } + var text string + if err := decoder.DecodeElement(&text, &value); err != nil { + return "", false + } + return text, true + } + } +} + +func launchdLastExitCode(output string) (int, bool) { + for _, line := range strings.Split(output, "\n") { + value, ok := strings.CutPrefix(strings.TrimSpace(line), "last exit code =") + if !ok { + continue + } + code, err := strconv.Atoi(strings.TrimSpace(value)) + return code, err == nil + } + return 0, false +} + func schedulerPaths(paths config.Paths, home string) []string { switch runtime.GOOS { case "darwin": - return []string{filepath.Join(home, "Library", "LaunchAgents", "io.tooltend.reconcile.plist")} + return []string{ + filepath.Join(home, "Library", "LaunchAgents", "io.tooltend.reconcile.plist"), + filepath.Join(home, "Library", "LaunchAgents", "io.tooltend.watchdog.plist"), + } case "linux": root := filepath.Join(home, ".config", "systemd", "user") - return []string{filepath.Join(root, "tooltend-reconcile.service"), filepath.Join(root, "tooltend-reconcile.timer")} + return []string{ + filepath.Join(root, "tooltend-reconcile.service"), filepath.Join(root, "tooltend-reconcile.timer"), + filepath.Join(root, "tooltend-watchdog.service"), filepath.Join(root, "tooltend-watchdog.timer"), + } default: return nil } } +func checkReconcileRun(ctx context.Context, database *store.Store, interval time.Duration, now time.Time) Check { + value, err := database.LatestReconcileRun(ctx) + if errors.Is(err, sql.ErrNoRows) { + return Check{Name: "reconcile_run", Level: LevelWarning, Message: "scheduled reconciliation has not completed yet"} + } + if err != nil { + return Check{Name: "reconcile_run", Level: LevelError, Message: "scheduled reconciliation history cannot be inspected"} + } + switch value.Status { + case "failed": + return Check{Name: "reconcile_run", Level: LevelError, Message: "the latest reconciliation failed with code " + value.ErrorCode} + case "running": + if now.Sub(value.StartedAt) > 15*time.Minute { + return Check{Name: "reconcile_run", Level: LevelError, Message: "the latest reconciliation appears stuck"} + } + return Check{Name: "reconcile_run", Level: LevelOK, Message: "reconciliation is currently running"} + case "incomplete": + return Check{Name: "reconcile_run", Level: LevelWarning, Message: "the latest reconciliation is waiting for a retry"} + case "succeeded": + if value.FinishedAt == nil { + return Check{Name: "reconcile_run", Level: LevelError, Message: "the latest reconciliation has an invalid completion record"} + } + if interval <= 0 { + interval = 24 * time.Hour + } + if now.Sub(*value.FinishedAt) > interval+2*time.Hour { + return Check{Name: "reconcile_run", Level: LevelWarning, Message: "the latest successful reconciliation is stale"} + } + return Check{Name: "reconcile_run", Level: LevelOK, Message: "the latest reconciliation completed successfully"} + default: + return Check{Name: "reconcile_run", Level: LevelError, Message: "the latest reconciliation has an invalid status"} + } +} + func RepairPlan(paths config.Paths) plan.Plan { return plan.Plan{ ID: "doctor-repair-v1", @@ -304,7 +484,7 @@ func RepairPlan(paths config.Paths) plan.Plan { if err := paths.Ensure(); err != nil { return err } - for _, dir := range []string{paths.ConfigDir, paths.StateDir, paths.DataDir, paths.ObjectsDir, paths.StagingDir, paths.GenerationsDir, paths.RuntimesDir} { + for _, dir := range []string{paths.ConfigDir, paths.StateDir, paths.LogsDir, paths.DataDir, paths.ObjectsDir, paths.StagingDir, paths.GenerationsDir, paths.RuntimesDir} { if err := os.Chmod(dir, 0o700); err != nil { return err } diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 66258d8..6dab951 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -6,11 +6,13 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/z2z23n0/tooltend/internal/config" "github.com/z2z23n0/tooltend/internal/execx" "github.com/z2z23n0/tooltend/internal/model" "github.com/z2z23n0/tooltend/internal/plan" + "github.com/z2z23n0/tooltend/internal/store" ) type fakeRunner struct{ calls int } @@ -85,13 +87,32 @@ Description=ToolTend one-shot reconciliation Type=oneshot Environment="PATH=/opt/tooltend/bin:/usr/bin:/bin" ExecStart="/opt/tooltend/bin/tooltend" reconcile --once --state-dir "/var/lib/tooltend-state" --json +` + watchdogService := `[Unit] +Description=Check ToolTend scheduled reconciliation + +[Service] +Type=oneshot +Environment="PATH=/opt/tooltend/bin:/usr/bin:/bin" +ExecStart="/opt/tooltend/bin/tooltend" watchdog --max-age 3h --state-dir "/var/lib/tooltend-state" --json ` plist := ` ProgramArguments /opt/tooltend/bin/tooltendreconcile--once ---state-dir/var/lib/tooltend-state +--state-dir/var/lib/tooltend-state--json EnvironmentVariablesPATH/opt/tooltend/bin:/usr/bin:/bin +StandardOutPath/var/lib/tooltend-state/logs/reconcile.stdout.log +StandardErrorPath/var/lib/tooltend-state/logs/reconcile.stderr.log +` + watchdogPlist := ` +ProgramArguments +/opt/tooltend/bin/tooltendwatchdog--max-age2h +--state-dir/var/lib/tooltend-state--json + +EnvironmentVariablesPATH/opt/tooltend/bin:/usr/bin:/bin +StandardOutPath/var/lib/tooltend-state/logs/watchdog.stdout.log +StandardErrorPath/var/lib/tooltend-state/logs/watchdog.stderr.log ` timer := `[Unit] Description=Run ToolTend reconciliation daily @@ -113,8 +134,11 @@ WantedBy=timers.target want bool }{ {name: "plist", file: "io.tooltend.reconcile.plist", content: plist, exe: executable, stateDir: stateDir, want: true}, + {name: "watchdog plist", file: "io.tooltend.watchdog.plist", content: watchdogPlist, exe: executable, stateDir: stateDir, want: true}, {name: "plist missing path", file: "io.tooltend.reconcile.plist", content: strings.Replace(plist, "PATH", "OLD_PATH", 1), exe: executable, stateDir: stateDir}, + {name: "plist wrong argv prefix", file: "io.tooltend.reconcile.plist", content: strings.Replace(plist, "/opt/tooltend/bin/tooltend", "/opt/tooltend/bin/tooltend-bundle-driver/opt/tooltend/bin/tooltend", 1), exe: executable, stateDir: stateDir}, {name: "service", file: "tooltend-reconcile.service", content: service, exe: executable, stateDir: stateDir, want: true}, + {name: "watchdog service", file: "tooltend-watchdog.service", content: watchdogService, exe: executable, stateDir: stateDir, want: true}, {name: "timer", file: "tooltend-reconcile.timer", content: timer, exe: executable, stateDir: stateDir, want: true}, {name: "timer missing calendar", file: "tooltend-reconcile.timer", content: strings.Replace(timer, "OnCalendar=", "Calendar=", 1), exe: executable, stateDir: stateDir}, {name: "timer missing persistence", file: "tooltend-reconcile.timer", content: strings.Replace(timer, "Persistent=true", "Persistent=false", 1), exe: executable, stateDir: stateDir}, @@ -133,6 +157,31 @@ WantedBy=timers.target } } +func TestLaunchdLastExitCode(t *testing.T) { + code, ok := launchdLastExitCode("state = not running\n\tlast exit code = 17\n") + if !ok || code != 17 { + t.Fatalf("code=%d ok=%v", code, ok) + } +} + +func TestReconcileRunCheckReportsFailureAndStaleness(t *testing.T) { + database, err := store.OpenRW(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + now := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC) + if check := checkReconcileRun(context.Background(), database, 24*time.Hour, now); check.Level != LevelWarning { + t.Fatalf("missing check = %#v", check) + } + if _, err := database.DB().Exec(`INSERT INTO reconcile_runs(id,reason,status,started_at,finished_at,error_code,summary_json) VALUES('run','scheduled','failed',?,?, 'task_failed','{}')`, now.Add(-time.Hour).Format(time.RFC3339Nano), now.Add(-59*time.Minute).Format(time.RFC3339Nano)); err != nil { + t.Fatal(err) + } + if check := checkReconcileRun(context.Background(), database, 24*time.Hour, now); check.Level != LevelError || !strings.Contains(check.Message, "task_failed") { + t.Fatalf("failed check = %#v", check) + } +} + func TestRepairPlanRefusesSchedulerFileChangedAfterPreview(t *testing.T) { home := t.TempDir() paths := config.ResolveWith(home, func(name string) string { diff --git a/internal/model/types.go b/internal/model/types.go index 470960c..b32d014 100644 --- a/internal/model/types.go +++ b/internal/model/types.go @@ -400,10 +400,21 @@ type HookEvent struct { type Notification struct { CandidateHash string `json:"candidate_hash"` Kind string `json:"kind"` + Message string `json:"message,omitempty"` QueuedAt time.Time `json:"queued_at"` ShownAt *time.Time `json:"shown_at,omitempty"` } +type ReconcileRun struct { + ID string `json:"id"` + Reason string `json:"reason"` + Status string `json:"status"` + StartedAt time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + SummaryJSON string `json:"summary_json"` +} + type Scan struct { ID string `json:"id"` Reason string `json:"reason"` diff --git a/internal/notify/desktop.go b/internal/notify/desktop.go new file mode 100644 index 0000000..5f7c332 --- /dev/null +++ b/internal/notify/desktop.go @@ -0,0 +1,47 @@ +package notify + +import ( + "context" + "errors" + "runtime" + "strings" + + "github.com/z2z23n0/tooltend/internal/execx" +) + +var ErrUnsupported = errors.New("desktop notifications are not supported on this platform") + +type Desktop struct { + GOOS string + Runner execx.Runner +} + +func (d Desktop) Send(ctx context.Context, title, message string) error { + if strings.TrimSpace(title) == "" || strings.TrimSpace(message) == "" { + return errors.New("desktop notification title and message are required") + } + goos := d.GOOS + if goos == "" { + goos = runtime.GOOS + } + runner := d.Runner + if runner == nil { + runner = execx.ExecRunner{} + } + switch goos { + case "darwin": + script := "display notification " + appleScriptString(message) + " with title " + appleScriptString(title) + _, err := runner.Run(ctx, "/usr/bin/osascript", "-e", script) + return err + case "linux": + _, err := runner.Run(ctx, "notify-send", "--app-name=ToolTend", title, message) + return err + default: + return ErrUnsupported + } +} + +func appleScriptString(value string) string { + value = strings.NewReplacer("\\", "\\\\", "\"", "\\\"", "\r", " ", "\n", " ").Replace(value) + return "\"" + value + "\"" +} diff --git a/internal/notify/desktop_test.go b/internal/notify/desktop_test.go new file mode 100644 index 0000000..c449c07 --- /dev/null +++ b/internal/notify/desktop_test.go @@ -0,0 +1,33 @@ +package notify + +import ( + "context" + "strings" + "testing" + + "github.com/z2z23n0/tooltend/internal/execx" +) + +type recordingRunner struct { + name string + args []string +} + +func (r *recordingRunner) Run(_ context.Context, name string, args ...string) (execx.Result, error) { + r.name, r.args = name, append([]string(nil), args...) + return execx.Result{}, nil +} + +func TestDarwinNotificationEscapesAppleScript(t *testing.T) { + runner := &recordingRunner{} + err := (Desktop{GOOS: "darwin", Runner: runner}).Send(context.Background(), `Tool"Tend`, "line 1\nline 2") + if err != nil { + t.Fatal(err) + } + if runner.name != "/usr/bin/osascript" || len(runner.args) != 2 || runner.args[0] != "-e" { + t.Fatalf("call = %s %#v", runner.name, runner.args) + } + if strings.Contains(runner.args[1], "\n") || !strings.Contains(runner.args[1], `Tool\"Tend`) { + t.Fatalf("unsafe script = %q", runner.args[1]) + } +} diff --git a/internal/reconcile/types.go b/internal/reconcile/types.go index b3fca57..8328fc9 100644 --- a/internal/reconcile/types.go +++ b/internal/reconcile/types.go @@ -78,26 +78,29 @@ type BindingResult struct { type FailureResult struct { BindingID string `json:"binding_id,omitempty"` + BundleID string `json:"bundle_id,omitempty"` TaskID string `json:"task_id,omitempty"` Code string `json:"code"` Retrying bool `json:"retrying,omitempty"` } type RunResult struct { - AlreadyRunning bool `json:"already_running"` - ScanID string `json:"scan_id,omitempty"` - StartedAt time.Time `json:"started_at"` - FinishedAt time.Time `json:"finished_at"` - Recovered int `json:"recovered_activations"` - BundleRecovery bundle.RecoveryResult `json:"bundle_recovery"` - Inventory inventory.PersistResult `json:"inventory"` - Scheduled int `json:"scheduled"` - Succeeded int `json:"succeeded"` - Retried int `json:"retried"` - Failed int `json:"failed"` - Skipped int `json:"skipped"` - Results []BindingResult `json:"results,omitempty"` - Failures []FailureResult `json:"failures,omitempty"` + AlreadyRunning bool `json:"already_running"` + FailureNotificationQueued bool `json:"failure_notification_queued,omitempty"` + RunID string `json:"run_id,omitempty"` + ScanID string `json:"scan_id,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + Recovered int `json:"recovered_activations"` + BundleRecovery bundle.RecoveryResult `json:"bundle_recovery"` + Inventory inventory.PersistResult `json:"inventory"` + Scheduled int `json:"scheduled"` + Succeeded int `json:"succeeded"` + Retried int `json:"retried"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` + Results []BindingResult `json:"results,omitempty"` + Failures []FailureResult `json:"failures,omitempty"` } // CodedError lets adapters expose a stable, non-sensitive reason code without diff --git a/internal/reconcile/worker.go b/internal/reconcile/worker.go index 2a0ed6f..b7314e9 100644 --- a/internal/reconcile/worker.go +++ b/internal/reconcile/worker.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "encoding/json" "errors" "fmt" "os" @@ -60,6 +61,75 @@ type Worker struct { } func (w *Worker) RunOnce(ctx context.Context, reason string) (RunResult, error) { + if err := w.validate(); err != nil { + return RunResult{}, err + } + reason = normalizeReason(reason) + started := w.now() + runID, err := model.NewID("run") + if err != nil { + return RunResult{}, err + } + if err := w.Database.BeginReconcileRun(ctx, model.ReconcileRun{ID: runID, Reason: reason, Status: "running", StartedAt: started}); err != nil { + return RunResult{}, fmt.Errorf("reconcile: begin run: %w", err) + } + + result, runErr := w.runOnce(ctx, reason) + result.RunID = runID + status, errorCode := "succeeded", "" + if result.AlreadyRunning { + status, errorCode = "incomplete", "already_running" + } else if runErr != nil { + status, errorCode = "failed", "reconcile_failed" + if code, _ := classifyError(runErr); code != "" { + errorCode = code + } + } else if result.Failed > 0 { + status, errorCode = "failed", "task_failed" + } else if result.Retried > 0 { + status, errorCode = "incomplete", "retry_pending" + } + summary, _ := json.Marshal(map[string]int{ + "scheduled": result.Scheduled, + "succeeded": result.Succeeded, + "retried": result.Retried, + "failed": result.Failed, + "skipped": result.Skipped, + }) + finished := w.now() + result.FinishedAt = finished + finishErr := w.Database.FinishReconcileRun(ctx, runID, status, errorCode, string(summary), finished) + var notificationErr error + if status == "failed" && w.Config.Notify.Mode != model.NotifyNone { + message := "ToolTend scheduled update failed" + if result.Failed > 0 { + message = fmt.Sprintf("ToolTend scheduled update failed: %d task(s) failed", result.Failed) + } + interval := w.Config.Check.Interval + if interval <= 0 { + interval = 24 * time.Hour + } + slot := started.UnixNano() / interval.Nanoseconds() + result.FailureNotificationQueued, notificationErr = w.Database.QueueNotification(ctx, model.Notification{ + CandidateHash: notificationHash("", strconv.FormatInt(slot, 10), errorCode), + Kind: "reconcile_failed:" + errorCode, + Message: message, + QueuedAt: finished, + }) + } + if runErr != nil { + return result, errors.Join(runErr, finishErr, notificationErr) + } + if finishErr != nil { + return result, fmt.Errorf("reconcile: finish run: %w", finishErr) + } + if notificationErr != nil { + return result, fmt.Errorf("reconcile: queue failure notification: %w", notificationErr) + } + return result, nil +} + +func (w *Worker) runOnce(ctx context.Context, reason string) (RunResult, error) { started := w.now() result := RunResult{StartedAt: started} finish := func() { result.FinishedAt = w.now() } @@ -206,6 +276,17 @@ func (w *Worker) RunOnce(ctx context.Context, reason string) (RunResult, error) } if inserted { result.Scheduled++ + } else { + var status model.TaskStatus + var code string + if err := w.Database.DB().QueryRowContext(ctx, `SELECT status,error_code FROM tasks WHERE id=?`, task.ID).Scan(&status, &code); err != nil { + finish() + return result, err + } + if status == model.TaskFailed { + result.Failed++ + result.Failures = append(result.Failures, FailureResult{BindingID: binding.ID, TaskID: task.ID, Code: code}) + } } } if err := markHookSignalsProcessed(ctx, w.Database, signal, started); err != nil { @@ -254,6 +335,16 @@ func (w *Worker) scheduleBundleTasks(ctx context.Context, signal int64, started } if inserted { result.Scheduled++ + } else { + var status model.TaskStatus + var code string + if err := w.Database.DB().QueryRowContext(ctx, `SELECT status,error_code FROM bundle_tasks WHERE id=?`, task.ID).Scan(&status, &code); err != nil { + return err + } + if status == model.TaskFailed { + result.Failed++ + result.Failures = append(result.Failures, FailureResult{BundleID: value.ID, TaskID: task.ID, Code: code}) + } } } return nil @@ -279,19 +370,22 @@ func (w *Worker) runBundleTasks(ctx context.Context, result *RunResult) error { } value, err := w.Database.GetBundle(ctx, task.BundleID) if err != nil { - _ = w.Database.FailBundleTask(ctx, task.ID, "bundle_unavailable") - result.Failed++ + if err := w.failBundleTask(ctx, task, model.Bundle{ID: task.BundleID}, "bundle_unavailable", now, result); err != nil { + return err + } continue } policy, err := w.Database.GetBundlePolicy(ctx, value.ID) if err != nil || value.ConfigState != model.BundleConfigured { - _ = w.Database.FailBundleTask(ctx, task.ID, "bundle_policy_unavailable") - result.Failed++ + if err := w.failBundleTask(ctx, task, value, "bundle_policy_unavailable", now, result); err != nil { + return err + } continue } if w.BundleCoordinator == nil { - _ = w.Database.FailBundleTask(ctx, task.ID, "bundle_coordinator_unavailable") - result.Failed++ + if err := w.failBundleTask(ctx, task, value, "bundle_coordinator_unavailable", now, result); err != nil { + return err + } continue } activate := task.Kind == "update" && policy.Mode == model.BundlePolicyAuto @@ -311,14 +405,36 @@ func (w *Worker) runBundleTasks(ctx context.Context, result *RunResult) error { result.Retried++ continue } - if err := w.Database.FailBundleTask(ctx, task.ID, code); err != nil { + if err := w.failBundleTask(ctx, task, value, code, now, result); err != nil { return err } - result.Failed++ } return nil } +func (w *Worker) failBundleTask(ctx context.Context, task model.BundleTask, value model.Bundle, code string, now time.Time, result *RunResult) error { + if err := w.Database.FailBundleTask(ctx, task.ID, code); err != nil { + return err + } + result.Failed++ + result.Failures = append(result.Failures, FailureResult{BundleID: task.BundleID, TaskID: task.ID, Code: code}) + if w.Config.Notify.Mode == model.NotifyNone { + return nil + } + name := strings.TrimSpace(value.Name) + if name == "" { + name = task.BundleID + } + kind := "bundle_failed:" + code + _, err := w.Database.QueueNotification(ctx, model.Notification{ + CandidateHash: notificationHash("", task.ID, kind), + Kind: kind, + Message: fmt.Sprintf("Bundle %s update failed (%s)", name, code), + QueuedAt: now, + }) + return err +} + func (w *Worker) bundleIdempotencyKey(value model.Bundle, policy model.BundlePolicy, signal int64, now time.Time) string { interval := w.Config.Check.Interval if interval <= 0 { diff --git a/internal/reconcile/worker_test.go b/internal/reconcile/worker_test.go index 50b70d9..d8ebd3b 100644 --- a/internal/reconcile/worker_test.go +++ b/internal/reconcile/worker_test.go @@ -145,6 +145,10 @@ func TestRunOnceStoresOnlyCodedFailureAndRetries(t *testing.T) { if first.Retried != 1 || first.Failed != 0 { t.Fatalf("unexpected retry result: %#v", first) } + firstRun, err := database.LatestReconcileRun(ctx) + if err != nil || firstRun.Status != "incomplete" || firstRun.ErrorCode != "retry_pending" { + t.Fatalf("retry run = %#v err=%v", firstRun, err) + } var code, summary, status string if err := database.DB().QueryRow(`SELECT error_code,error_summary,status FROM tasks WHERE binding_id='retry'`).Scan(&code, &summary, &status); err != nil { t.Fatal(err) @@ -167,6 +171,65 @@ func TestRunOnceStoresOnlyCodedFailureAndRetries(t *testing.T) { } } +func TestRunOnceRecordsAndNotifiesTerminalBundleFailure(t *testing.T) { + ctx := context.Background() + database, paths := openWorkerStore(t) + now := time.Date(2026, 7, 18, 3, 0, 0, 0, time.UTC) + bundleValue := model.Bundle{ + ID: "bundle-mainline", Slug: "mainline", Name: "Mainline", RecipeID: "mainline", RecipeVersion: "1", + RecipeSource: "local", Owner: model.LifecycleDelegated, ConfigState: model.BundleUnconfigured, + Confidence: model.BundleConfidenceHigh, MetadataJSON: `{}`, DiscoveredAt: now, LastSeenAt: now, + } + if err := database.UpsertBundle(ctx, bundleValue); err != nil { + t.Fatal(err) + } + if err := database.ConfigureBundle(ctx, model.BundlePolicy{BundleID: bundleValue.ID, Mode: model.BundlePolicyAuto, RecipeTrusted: true, UpdatedAt: now}); err != nil { + t.Fatal(err) + } + worker := Worker{ + Database: database, Paths: paths, Config: config.Default(), Now: func() time.Time { return now }, + Recover: func(context.Context, *store.Store, config.Paths) (int, error) { return 0, nil }, + Inventory: func(context.Context, *store.Store, InventoryOptions) (inventory.PersistResult, error) { + return inventory.PersistResult{}, nil + }, + Coordinator: CoordinatorFunc(func(context.Context, Request) (Outcome, error) { + return Outcome{}, nil + }), + BundleCoordinator: BundleCoordinatorFunc(func(context.Context, model.Bundle, model.BundlePolicy, bool) error { + return NewCodedError("release_manifest_invalid", false) + }), + } + result, err := worker.RunOnce(ctx, ReasonScheduled) + if err != nil { + t.Fatal(err) + } + if result.Failed != 1 || len(result.Failures) != 1 || result.Failures[0].BundleID != bundleValue.ID || !result.FailureNotificationQueued { + t.Fatalf("result = %#v", result) + } + run, err := database.LatestReconcileRun(ctx) + if err != nil || run.Status != "failed" || run.ErrorCode != "task_failed" { + t.Fatalf("run = %#v err=%v", run, err) + } + var bundleMessages, runMessages int + if err := database.DB().QueryRow(`SELECT COUNT(*) FROM notifications WHERE message LIKE 'Bundle Mainline%'`).Scan(&bundleMessages); err != nil { + t.Fatal(err) + } + if err := database.DB().QueryRow(`SELECT COUNT(*) FROM notifications WHERE message LIKE 'ToolTend scheduled update failed%'`).Scan(&runMessages); err != nil { + t.Fatal(err) + } + if bundleMessages != 1 || runMessages != 1 { + t.Fatalf("bundle notifications=%d run notifications=%d", bundleMessages, runMessages) + } + second, err := worker.RunOnce(ctx, ReasonKick) + if err != nil || second.Failed != 1 || second.FailureNotificationQueued { + t.Fatalf("failed task was hidden by idempotent rerun: result=%#v err=%v", second, err) + } + var notifications int + if err := database.DB().QueryRow(`SELECT COUNT(*) FROM notifications`).Scan(¬ifications); err != nil || notifications != 2 { + t.Fatalf("notifications=%d err=%v", notifications, err) + } +} + func TestClassifySanitizedUpstreamFailuresAsRetryable(t *testing.T) { for _, err := range []error{ errors.New("lifecycle: resolve update: npm version lookup failed"), @@ -208,6 +271,10 @@ func TestRunOnceNonBlockingLockLeavesActiveMarker(t *testing.T) { if !result.AlreadyRunning { t.Fatalf("expected non-blocking lock result: %#v", result) } + run, err := database.LatestReconcileRun(context.Background()) + if err != nil || run.Status != "incomplete" || run.ErrorCode != "already_running" { + t.Fatalf("run = %#v err=%v", run, err) + } if _, err := os.Stat(marker); err != nil { t.Fatalf("active worker marker was cleared: %v", err) } diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index f4619f9..c496f3e 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -11,6 +11,7 @@ import ( "runtime" "strconv" "strings" + "time" "github.com/z2z23n0/tooltend/internal/execx" "github.com/z2z23n0/tooltend/internal/safeio" @@ -23,8 +24,9 @@ type File struct { } type Plan struct { - Platform string `json:"platform"` - Files []File `json:"files"` + Platform string `json:"platform"` + Files []File `json:"files"` + Directories []string `json:"directories,omitempty"` } type Options struct { @@ -45,13 +47,18 @@ func BuildPlan(options Options) (Plan, error) { } switch runtime.GOOS { case "darwin": - path := filepath.Join(options.Home, "Library", "LaunchAgents", "io.tooltend.reconcile.plist") - return Plan{Platform: "launchd", Files: []File{{Path: path, Content: []byte(renderLaunchd(options)), Mode: 0o600}}}, nil + root := filepath.Join(options.Home, "Library", "LaunchAgents") + return Plan{Platform: "launchd", Directories: []string{filepath.Join(options.StateDir, "logs")}, Files: []File{ + {Path: filepath.Join(root, "io.tooltend.reconcile.plist"), Content: []byte(renderLaunchd(options)), Mode: 0o600}, + {Path: filepath.Join(root, "io.tooltend.watchdog.plist"), Content: []byte(renderLaunchdWatchdog(options)), Mode: 0o600}, + }}, nil case "linux": root := filepath.Join(options.Home, ".config", "systemd", "user") return Plan{Platform: "systemd", Files: []File{ {Path: filepath.Join(root, "tooltend-reconcile.service"), Content: []byte(renderSystemdService(options)), Mode: 0o600}, {Path: filepath.Join(root, "tooltend-reconcile.timer"), Content: []byte(renderSystemdTimer(options)), Mode: 0o600}, + {Path: filepath.Join(root, "tooltend-watchdog.service"), Content: []byte(renderSystemdWatchdogService(options)), Mode: 0o600}, + {Path: filepath.Join(root, "tooltend-watchdog.timer"), Content: []byte(renderSystemdWatchdogTimer(options)), Mode: 0o600}, }}, nil default: return Plan{}, fmt.Errorf("daily scheduling is not supported on %s", runtime.GOOS) @@ -59,6 +66,14 @@ func BuildPlan(options Options) (Plan, error) { } func Apply(plan Plan) error { + for _, directory := range plan.Directories { + if err := os.MkdirAll(directory, 0o700); err != nil { + return err + } + if err := os.Chmod(directory, 0o700); err != nil { + return err + } + } for _, file := range plan.Files { if err := safeio.AtomicWriteFile(file.Path, file.Content, file.Mode); err != nil { return err @@ -76,25 +91,27 @@ func Activate(ctx context.Context, schedule Plan, runner execx.Runner) error { } switch schedule.Platform { case "launchd": - if len(schedule.Files) != 1 || filepath.Base(schedule.Files[0].Path) != "io.tooltend.reconcile.plist" { + if !hasExactFiles(schedule.Files, "io.tooltend.reconcile.plist", "io.tooltend.watchdog.plist") { return errors.New("scheduler: invalid launchd plan") } domain := "gui/" + strconv.Itoa(os.Getuid()) - // bootout is intentionally best-effort: a first installation has no - // existing job, while a repair must replace an already loaded plist. - _, _ = runner.Run(ctx, "launchctl", "bootout", domain, schedule.Files[0].Path) - if _, err := runner.Run(ctx, "launchctl", "bootstrap", domain, schedule.Files[0].Path); err != nil { - return fmt.Errorf("scheduler: activate launchd job: %w", err) + for _, file := range schedule.Files { + // bootout is intentionally best-effort: a first installation has no + // existing job, while a repair must replace an already loaded plist. + _, _ = runner.Run(ctx, "launchctl", "bootout", domain, file.Path) + if _, err := runner.Run(ctx, "launchctl", "bootstrap", domain, file.Path); err != nil { + return fmt.Errorf("scheduler: activate launchd job: %w", err) + } } return nil case "systemd": - if len(schedule.Files) != 2 { + if !hasExactFiles(schedule.Files, "tooltend-reconcile.service", "tooltend-reconcile.timer", "tooltend-watchdog.service", "tooltend-watchdog.timer") { return errors.New("scheduler: invalid systemd plan") } if _, err := runner.Run(ctx, "systemctl", "--user", "daemon-reload"); err != nil { return fmt.Errorf("scheduler: reload systemd user units: %w", err) } - if _, err := runner.Run(ctx, "systemctl", "--user", "enable", "--now", "tooltend-reconcile.timer"); err != nil { + if _, err := runner.Run(ctx, "systemctl", "--user", "enable", "--now", "tooltend-reconcile.timer", "tooltend-watchdog.timer"); err != nil { return fmt.Errorf("scheduler: activate systemd timer: %w", err) } return nil @@ -112,20 +129,47 @@ func Deactivate(ctx context.Context, schedule Plan, runner execx.Runner) error { } switch schedule.Platform { case "launchd": - if len(schedule.Files) != 1 { + if !hasExactFiles(schedule.Files, "io.tooltend.reconcile.plist", "io.tooltend.watchdog.plist") { return errors.New("scheduler: invalid launchd plan") } domain := "gui/" + strconv.Itoa(os.Getuid()) - _, err := runner.Run(ctx, "launchctl", "bootout", domain, schedule.Files[0].Path) - return err + var result error + for _, file := range schedule.Files { + _, err := runner.Run(ctx, "launchctl", "bootout", domain, file.Path) + if filepath.Base(file.Path) == "io.tooltend.reconcile.plist" { + result = errors.Join(result, err) + } + } + return result case "systemd": - _, err := runner.Run(ctx, "systemctl", "--user", "disable", "--now", "tooltend-reconcile.timer") + if !hasExactFiles(schedule.Files, "tooltend-reconcile.service", "tooltend-reconcile.timer", "tooltend-watchdog.service", "tooltend-watchdog.timer") { + return errors.New("scheduler: invalid systemd plan") + } + _, err := runner.Run(ctx, "systemctl", "--user", "disable", "--now", "tooltend-reconcile.timer", "tooltend-watchdog.timer") return err default: return fmt.Errorf("scheduler: unsupported plan platform %q", schedule.Platform) } } +func hasExactFiles(files []File, names ...string) bool { + if len(files) != len(names) { + return false + } + expected := make(map[string]struct{}, len(names)) + for _, name := range names { + expected[name] = struct{}{} + } + for _, file := range files { + name := filepath.Base(file.Path) + if _, ok := expected[name]; !ok { + return false + } + delete(expected, name) + } + return len(expected) == 0 +} + func randomDailyTime() (int, int) { var value uint16 if err := binary.Read(rand.Reader, binary.LittleEndian, &value); err != nil { @@ -137,7 +181,18 @@ func randomDailyTime() (int, int) { func renderLaunchd(options Options) string { args := []string{options.Executable, "reconcile", "--once", "--state-dir", options.StateDir, "--json"} + return renderLaunchdJob("io.tooltend.reconcile", args, options, options.Hour, options.Minute, "reconcile") +} + +func renderLaunchdWatchdog(options Options) string { + hour, minute := watchdogTime(options.Hour, options.Minute) + args := []string{options.Executable, "watchdog", "--max-age", "2h", "--state-dir", options.StateDir, "--json"} + return renderLaunchdJob("io.tooltend.watchdog", args, options, hour, minute, "watchdog") +} + +func renderLaunchdJob(label string, args []string, options Options, hour, minute int, logName string) string { pathEnv := workerPATH(options.Executable, options.PathEnv) + logsDir := filepath.Join(options.StateDir, "logs") var program strings.Builder for _, arg := range args { program.WriteString(" ") @@ -148,7 +203,7 @@ func renderLaunchd(options Options) string { - Labelio.tooltend.reconcile + Label` + xmlEscape(label) + ` ProgramArguments ` + program.String() + ` @@ -158,13 +213,13 @@ func renderLaunchd(options Options) string { StartCalendarInterval - Hour` + strconv.Itoa(options.Hour) + ` - Minute` + strconv.Itoa(options.Minute) + ` + Hour` + strconv.Itoa(hour) + ` + Minute` + strconv.Itoa(minute) + ` ProcessTypeBackground LowPriorityIO - StandardOutPath/dev/null - StandardErrorPath/dev/null + StandardOutPath` + xmlEscape(filepath.Join(logsDir, logName+".stdout.log")) + ` + StandardErrorPath` + xmlEscape(filepath.Join(logsDir, logName+".stderr.log")) + ` ` @@ -181,6 +236,17 @@ ExecStart=` + systemdQuote(options.Executable) + ` reconcile --once --state-dir ` } +func renderSystemdWatchdogService(options Options) string { + return `[Unit] +Description=Check ToolTend scheduled reconciliation + +[Service] +Type=oneshot +Environment=` + systemdQuote("PATH="+workerPATH(options.Executable, options.PathEnv)) + ` +ExecStart=` + systemdQuote(options.Executable) + ` watchdog --max-age 3h --state-dir ` + systemdQuote(options.StateDir) + ` --json +` +} + func workerPATH(executable, current string) string { if strings.TrimSpace(current) == "" { current = os.Getenv("PATH") @@ -208,11 +274,20 @@ func workerPATH(executable, current string) string { } func renderSystemdTimer(options Options) string { + return renderSystemdTimerAt("Run ToolTend reconciliation daily", options.Hour, options.Minute) +} + +func renderSystemdWatchdogTimer(options Options) string { + hour, minute := timeAfter(options.Hour, options.Minute, 2*time.Hour) + return renderSystemdTimerAt("Check ToolTend reconciliation daily", hour, minute) +} + +func renderSystemdTimerAt(description string, hour, minute int) string { return `[Unit] -Description=Run ToolTend reconciliation daily +Description=` + description + ` [Timer] -OnCalendar=*-*-* ` + fmt.Sprintf("%02d:%02d:00", options.Hour, options.Minute) + ` +OnCalendar=*-*-* ` + fmt.Sprintf("%02d:%02d:00", hour, minute) + ` RandomizedDelaySec=1h Persistent=true @@ -221,6 +296,15 @@ WantedBy=timers.target ` } +func watchdogTime(hour, minute int) (int, int) { + return timeAfter(hour, minute, time.Hour) +} + +func timeAfter(hour, minute int, offset time.Duration) (int, int) { + minutes := (hour*60 + minute + int(offset/time.Minute)) % (24 * 60) + return minutes / 60, minutes % 60 +} + func systemdQuote(value string) string { return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(value) + `"` } diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index fe84fce..3d3e699 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -60,6 +60,30 @@ func TestRenderedSchedulesIncludeWorkerPATH(t *testing.T) { } } +func TestRenderedLaunchdSchedulesPersistLogsAndRunWatchdogLater(t *testing.T) { + options := Options{Executable: "/Users/example/.local/bin/tooltend", StateDir: "/Users/example/.local/state/tooltend", Hour: 23, Minute: 40} + reconcile := renderLaunchd(options) + watchdog := renderLaunchdWatchdog(options) + for _, value := range []string{"reconcile.stdout.log", "reconcile.stderr.log"} { + if !strings.Contains(reconcile, value) { + t.Fatalf("reconcile schedule missing %s: %s", value, reconcile) + } + } + if !strings.Contains(watchdog, "watchdog") || !strings.Contains(watchdog, "watchdog.stderr.log") || + !strings.Contains(watchdog, "Hour0") || !strings.Contains(watchdog, "Minute40") { + t.Fatalf("invalid watchdog schedule: %s", watchdog) + } +} + +func TestSystemdWatchdogWaitsPastReconcileJitter(t *testing.T) { + options := Options{Executable: "/usr/local/bin/tooltend", StateDir: "/var/lib/tooltend", Hour: 23, Minute: 40} + service := renderSystemdWatchdogService(options) + timer := renderSystemdWatchdogTimer(options) + if !strings.Contains(service, "watchdog --max-age 3h") || !strings.Contains(timer, "OnCalendar=*-*-* 01:40:00") { + t.Fatalf("service=%s\ntimer=%s", service, timer) + } +} + type recordingRunner struct { calls []string fail string @@ -76,16 +100,16 @@ func (r *recordingRunner) Run(_ context.Context, name string, args ...string) (e func TestActivateRegistersOneShotSchedule(t *testing.T) { runner := &recordingRunner{} - launchd := Plan{Platform: "launchd", Files: []File{{Path: "/tmp/io.tooltend.reconcile.plist"}}} + launchd := Plan{Platform: "launchd", Files: []File{{Path: "/tmp/io.tooltend.reconcile.plist"}, {Path: "/tmp/io.tooltend.watchdog.plist"}}} if err := Activate(context.Background(), launchd, runner); err != nil { t.Fatal(err) } - if len(runner.calls) != 2 || !strings.Contains(runner.calls[0], "bootout") || !strings.Contains(runner.calls[1], "bootstrap") { + if len(runner.calls) != 4 || !strings.Contains(runner.calls[0], "bootout") || !strings.Contains(runner.calls[1], "bootstrap") || !strings.Contains(runner.calls[3], "watchdog") { t.Fatalf("launchd calls = %#v", runner.calls) } runner.calls = nil - systemd := Plan{Platform: "systemd", Files: []File{{Path: "/tmp/tooltend-reconcile.service"}, {Path: "/tmp/tooltend-reconcile.timer"}}} + systemd := Plan{Platform: "systemd", Files: []File{{Path: "/tmp/tooltend-reconcile.service"}, {Path: "/tmp/tooltend-reconcile.timer"}, {Path: "/tmp/tooltend-watchdog.service"}, {Path: "/tmp/tooltend-watchdog.timer"}}} if err := Activate(context.Background(), systemd, runner); err != nil { t.Fatal(err) } @@ -98,4 +122,7 @@ func TestActivateFailsClosedForMalformedPlan(t *testing.T) { if err := Activate(context.Background(), Plan{Platform: "launchd"}, &recordingRunner{}); err == nil { t.Fatal("expected malformed launchd plan to be rejected") } + if err := Activate(context.Background(), Plan{Platform: "launchd", Files: []File{{Path: "/tmp/io.tooltend.reconcile.plist"}}}, &recordingRunner{}); err == nil { + t.Fatal("expected launchd plan without watchdog to be rejected") + } } diff --git a/internal/store/bundles_test.go b/internal/store/bundles_test.go index 6f53f53..9f2af2b 100644 --- a/internal/store/bundles_test.go +++ b/internal/store/bundles_test.go @@ -9,7 +9,7 @@ import ( "testing" ) -func TestSchemaV5MigratesV4WithBackup(t *testing.T) { +func TestSchemaV6MigratesV4WithBackup(t *testing.T) { path := filepath.Join(t.TempDir(), "state.db") db, err := open(path, "rwc", 0, false) if err != nil { @@ -45,14 +45,14 @@ func TestSchemaV5MigratesV4WithBackup(t *testing.T) { } defer database.Close() version, err := database.UserVersion(context.Background()) - if err != nil || version != 5 { + if err != nil || version != 6 { t.Fatalf("version=%d err=%v", version, err) } var tables int - if err := database.DB().QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('bundles','bundle_releases','bundle_artifacts','installations','consumer_bindings','bundle_policies','bundle_transactions','bundle_transaction_steps','bundle_receipts','bundle_health_checks','bundle_tasks')`).Scan(&tables); err != nil { + if err := database.DB().QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('bundles','bundle_releases','bundle_artifacts','installations','consumer_bindings','bundle_policies','bundle_transactions','bundle_transaction_steps','bundle_receipts','bundle_health_checks','bundle_tasks','reconcile_runs')`).Scan(&tables); err != nil { t.Fatal(err) } - if tables != 11 { + if tables != 12 { t.Fatalf("bundle tables = %d", tables) } backups, err := filepath.Glob(path + ".backup-v4-*") diff --git a/internal/store/lifecycle.go b/internal/store/lifecycle.go index 2f3f98d..01fe9e4 100644 --- a/internal/store/lifecycle.go +++ b/internal/store/lifecycle.go @@ -627,7 +627,7 @@ func (s *Store) RecordHookEvent(ctx context.Context, value model.HookEvent) (int } func (s *Store) QueueNotification(ctx context.Context, value model.Notification) (bool, error) { - result, err := s.db.ExecContext(ctx, `INSERT INTO notifications(candidate_hash,kind,queued_at,shown_at) VALUES(?,?,?,?) ON CONFLICT(candidate_hash,kind) DO NOTHING`, value.CandidateHash, value.Kind, timeText(value.QueuedAt), nullableTimeText(value.ShownAt)) + result, err := s.db.ExecContext(ctx, `INSERT INTO notifications(candidate_hash,kind,message,queued_at,shown_at) VALUES(?,?,?,?,?) ON CONFLICT(candidate_hash,kind) DO NOTHING`, value.CandidateHash, value.Kind, value.Message, timeText(value.QueuedAt), nullableTimeText(value.ShownAt)) if err != nil { return false, err } @@ -635,13 +635,19 @@ func (s *Store) QueueNotification(ctx context.Context, value model.Notification) return count == 1, err } +func (s *Store) HasNotificationKindSince(ctx context.Context, kindPrefix string, since time.Time) (bool, error) { + var exists int + err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM notifications WHERE kind LIKE ? AND queued_at>=?)`, kindPrefix+"%", timeText(since)).Scan(&exists) + return exists == 1, err +} + func (s *Store) TakeNotifications(ctx context.Context, limit int) ([]model.Notification, error) { if limit <= 0 { limit = 20 } result := []model.Notification{} err := s.WithTx(ctx, func(tx *sql.Tx) error { - rows, err := tx.QueryContext(ctx, `SELECT candidate_hash,kind,queued_at FROM notifications WHERE shown_at IS NULL ORDER BY queued_at LIMIT ?`, limit) + rows, err := tx.QueryContext(ctx, `SELECT candidate_hash,kind,message,queued_at FROM notifications WHERE shown_at IS NULL ORDER BY queued_at LIMIT ?`, limit) if err != nil { return err } @@ -649,7 +655,7 @@ func (s *Store) TakeNotifications(ctx context.Context, limit int) ([]model.Notif for rows.Next() { var value model.Notification var queued string - if err := rows.Scan(&value.CandidateHash, &value.Kind, &queued); err != nil { + if err := rows.Scan(&value.CandidateHash, &value.Kind, &value.Message, &queued); err != nil { return err } value.QueuedAt, err = parseTime(queued) diff --git a/internal/store/migrations/0006_reconcile_runs.sql b/internal/store/migrations/0006_reconcile_runs.sql new file mode 100644 index 0000000..b750681 --- /dev/null +++ b/internal/store/migrations/0006_reconcile_runs.sql @@ -0,0 +1,13 @@ +ALTER TABLE notifications ADD COLUMN message TEXT NOT NULL DEFAULT ''; + +CREATE TABLE reconcile_runs ( + id TEXT PRIMARY KEY, + reason TEXT NOT NULL CHECK (reason IN ('scheduled','kick','command')), + status TEXT NOT NULL CHECK (status IN ('running','succeeded','incomplete','failed')), + started_at TEXT NOT NULL, + finished_at TEXT, + error_code TEXT NOT NULL DEFAULT '', + summary_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(summary_json)) +); + +CREATE INDEX idx_reconcile_runs_started ON reconcile_runs(started_at DESC); diff --git a/internal/store/reconcile_runs.go b/internal/store/reconcile_runs.go new file mode 100644 index 0000000..d5e15f6 --- /dev/null +++ b/internal/store/reconcile_runs.go @@ -0,0 +1,64 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/z2z23n0/tooltend/internal/model" +) + +func (s *Store) BeginReconcileRun(ctx context.Context, value model.ReconcileRun) error { + if value.ID == "" || value.Status != "running" { + return errors.New("store: reconcile run must start in running state") + } + _, err := s.db.ExecContext(ctx, `INSERT INTO reconcile_runs(id,reason,status,started_at,finished_at,error_code,summary_json) VALUES(?,?,?,?,NULL,'','{}')`, value.ID, value.Reason, value.Status, timeText(value.StartedAt)) + return err +} + +func (s *Store) FinishReconcileRun(ctx context.Context, id, status, errorCode, summaryJSON string, finishedAt time.Time) error { + if status != "succeeded" && status != "incomplete" && status != "failed" { + return fmt.Errorf("store: invalid reconcile run status %q", status) + } + if summaryJSON == "" { + summaryJSON = "{}" + } + result, err := s.db.ExecContext(ctx, `UPDATE reconcile_runs SET status=?,finished_at=?,error_code=?,summary_json=? WHERE id=? AND status='running'`, status, timeText(finishedAt), errorCode, summaryJSON, id) + if err != nil { + return err + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count != 1 { + return sql.ErrNoRows + } + return nil +} + +func (s *Store) LatestReconcileRun(ctx context.Context) (model.ReconcileRun, error) { + var value model.ReconcileRun + var started string + var finished sql.NullString + err := s.db.QueryRowContext(ctx, `SELECT id,reason,status,started_at,finished_at,error_code,summary_json FROM reconcile_runs ORDER BY COALESCE(finished_at,started_at) DESC,started_at DESC LIMIT 1`).Scan( + &value.ID, &value.Reason, &value.Status, &started, &finished, &value.ErrorCode, &value.SummaryJSON, + ) + if err != nil { + return model.ReconcileRun{}, err + } + value.StartedAt, err = parseTime(started) + if err != nil { + return model.ReconcileRun{}, err + } + if finished.Valid { + parsed, parseErr := parseTime(finished.String) + if parseErr != nil { + return model.ReconcileRun{}, parseErr + } + value.FinishedAt = &parsed + } + return value, nil +} diff --git a/internal/store/store.go b/internal/store/store.go index 80d98c3..6d22477 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,7 +17,7 @@ import ( _ "modernc.org/sqlite" ) -const SchemaVersion = 5 +const SchemaVersion = 6 //go:embed migrations/*.sql var migrationFiles embed.FS diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 7b1cc7c..7f9c6e4 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -163,7 +163,7 @@ func TestTaskAndNotificationDeduplication(t *testing.T) { t.Fatal(err) } hash := strings.Repeat("c", 64) - queued, err := s.QueueNotification(ctx, model.Notification{CandidateHash: hash, Kind: "failure", QueuedAt: now}) + queued, err := s.QueueNotification(ctx, model.Notification{CandidateHash: hash, Kind: "failure", Message: "更新失败", QueuedAt: now}) if err != nil || !queued { t.Fatalf("queue: %v %v", queued, err) } @@ -178,6 +178,13 @@ func TestTaskAndNotificationDeduplication(t *testing.T) { if len(notifications) != 1 { t.Fatalf("notifications = %d", len(notifications)) } + if notifications[0].Message != "更新失败" { + t.Fatalf("notification message = %q", notifications[0].Message) + } + hasFailure, err := s.HasNotificationKindSince(ctx, "fail", now.Add(-time.Second)) + if err != nil || !hasFailure { + t.Fatalf("has failure=%v err=%v", hasFailure, err) + } notifications, err = s.TakeNotifications(ctx, 10) if err != nil { t.Fatal(err) @@ -187,6 +194,25 @@ func TestTaskAndNotificationDeduplication(t *testing.T) { } } +func TestReconcileRunRecordsTerminalOutcome(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + started := time.Date(2026, 7, 18, 3, 0, 0, 0, time.UTC) + if err := s.BeginReconcileRun(ctx, model.ReconcileRun{ID: "run-1", Reason: "scheduled", Status: "running", StartedAt: started}); err != nil { + t.Fatal(err) + } + if err := s.FinishReconcileRun(ctx, "run-1", "failed", "bundle_failed", `{"failed":1}`, started.Add(time.Minute)); err != nil { + t.Fatal(err) + } + got, err := s.LatestReconcileRun(ctx) + if err != nil { + t.Fatal(err) + } + if got.ID != "run-1" || got.Status != "failed" || got.ErrorCode != "bundle_failed" || got.FinishedAt == nil || got.SummaryJSON != `{"failed":1}` { + t.Fatalf("run = %#v", got) + } +} + func TestHookEventDoesNotHaveRawSensitiveColumns(t *testing.T) { s := openTestStore(t) ctx := context.Background() diff --git a/internal/watchdog/watchdog.go b/internal/watchdog/watchdog.go new file mode 100644 index 0000000..7d2d1ca --- /dev/null +++ b/internal/watchdog/watchdog.go @@ -0,0 +1,113 @@ +package watchdog + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/z2z23n0/tooltend/internal/model" + "github.com/z2z23n0/tooltend/internal/store" +) + +type Notifier interface { + Send(context.Context, string, string) error +} + +type Service struct { + Database *store.Store + Notifier Notifier + Now func() time.Time + Enabled bool +} + +type Result struct { + Healthy bool `json:"healthy"` + Alerted bool `json:"alerted"` + DesktopNotified bool `json:"desktop_notified"` + Reason string `json:"reason,omitempty"` + LatestRun *model.ReconcileRun `json:"latest_run,omitempty"` +} + +func (s Service) Check(ctx context.Context, maxAge time.Duration) (Result, error) { + if s.Database == nil || s.Database.DB() == nil { + return Result{}, fmt.Errorf("watchdog: database is required") + } + if maxAge <= 0 { + return Result{}, fmt.Errorf("watchdog: max age must be positive") + } + now := time.Now() + if s.Now != nil { + now = s.Now() + } + latest, err := s.Database.LatestReconcileRun(ctx) + result := Result{} + switch { + case err == nil && latest.Status == "succeeded" && latest.FinishedAt != nil && !latest.FinishedAt.Before(now.Add(-maxAge)): + result.Healthy, result.LatestRun = true, &latest + return result, nil + case err != nil && !isNoRows(err): + return result, err + case err != nil: + result.Reason = "missing_run" + case latest.Status == "failed": + result.Reason, result.LatestRun = "failed_run", &latest + case latest.Status == "running": + result.Reason, result.LatestRun = "unfinished_run", &latest + case latest.Status == "incomplete": + result.Reason, result.LatestRun = "unfinished_run", &latest + default: + result.Reason, result.LatestRun = "stale_run", &latest + } + if !s.Enabled { + return result, nil + } + if result.Reason == "failed_run" { + alreadyNotified, err := s.Database.HasNotificationKindSince(ctx, "reconcile_failed:", startOfDay(now)) + if err != nil { + return result, err + } + if alreadyNotified { + return result, nil + } + } + message := watchdogMessage(result.Reason) + hash := sha256.Sum256([]byte("watchdog\x00" + now.Format("2006-01-02") + "\x00" + result.Reason)) + queued, err := s.Database.QueueNotification(ctx, model.Notification{ + CandidateHash: hex.EncodeToString(hash[:]), + Kind: "watchdog:" + result.Reason, + Message: message, + QueuedAt: now.UTC(), + }) + if err != nil { + return result, err + } + result.Alerted = queued + if queued && s.Notifier != nil { + result.DesktopNotified = s.Notifier.Send(ctx, "ToolTend", message) == nil + } + return result, nil +} + +func watchdogMessage(reason string) string { + switch reason { + case "failed_run": + return "Scheduled update failed. Run `tooltend doctor` for details." + case "unfinished_run": + return "Scheduled update did not finish. Run `tooltend doctor` for details." + case "stale_run": + return "Scheduled update has not completed recently. Run `tooltend doctor` for details." + default: + return "Scheduled update did not run. Run `tooltend doctor` for details." + } +} + +func isNoRows(err error) bool { return errors.Is(err, sql.ErrNoRows) } + +func startOfDay(value time.Time) time.Time { + year, month, day := value.Date() + return time.Date(year, month, day, 0, 0, 0, 0, value.Location()) +} diff --git a/internal/watchdog/watchdog_test.go b/internal/watchdog/watchdog_test.go new file mode 100644 index 0000000..7535b8a --- /dev/null +++ b/internal/watchdog/watchdog_test.go @@ -0,0 +1,89 @@ +package watchdog + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/z2z23n0/tooltend/internal/model" + "github.com/z2z23n0/tooltend/internal/store" +) + +type recordingNotifier struct { + calls int +} + +func (n *recordingNotifier) Send(context.Context, string, string) error { + n.calls++ + return nil +} + +func TestMissingRunAlertsOnlyOncePerDay(t *testing.T) { + database, err := store.OpenRW(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + now := time.Date(2026, 7, 18, 20, 30, 0, 0, time.Local) + notifier := &recordingNotifier{} + service := Service{Database: database, Notifier: notifier, Now: func() time.Time { return now }, Enabled: true} + first, err := service.Check(context.Background(), 2*time.Hour) + if err != nil { + t.Fatal(err) + } + second, err := service.Check(context.Background(), 2*time.Hour) + if err != nil { + t.Fatal(err) + } + if first.Healthy || !first.Alerted || !first.DesktopNotified || second.Alerted || notifier.calls != 1 { + t.Fatalf("first=%#v second=%#v calls=%d", first, second, notifier.calls) + } +} + +func TestRecentSuccessfulRunIsHealthy(t *testing.T) { + database, err := store.OpenRW(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + now := time.Date(2026, 7, 18, 20, 30, 0, 0, time.UTC) + if err := database.BeginReconcileRun(context.Background(), model.ReconcileRun{ID: "run", Reason: "scheduled", Status: "running", StartedAt: now.Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := database.FinishReconcileRun(context.Background(), "run", "succeeded", "", `{}`, now.Add(-59*time.Minute)); err != nil { + t.Fatal(err) + } + result, err := (Service{Database: database, Enabled: true, Now: func() time.Time { return now }}).Check(context.Background(), 2*time.Hour) + if err != nil || !result.Healthy || result.Alerted { + t.Fatalf("result=%#v err=%v", result, err) + } +} + +func TestFailedRunDoesNotDuplicateReconcileFailureAlert(t *testing.T) { + database, err := store.OpenRW(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + ctx := context.Background() + now := time.Date(2026, 7, 18, 20, 30, 0, 0, time.Local) + if err := database.BeginReconcileRun(ctx, model.ReconcileRun{ID: "run", Reason: "scheduled", Status: "running", StartedAt: now.Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := database.FinishReconcileRun(ctx, "run", "failed", "task_failed", `{}`, now.Add(-59*time.Minute)); err != nil { + t.Fatal(err) + } + queued, err := database.QueueNotification(ctx, model.Notification{ + CandidateHash: strings.Repeat("f", 64), Kind: "reconcile_failed:task_failed", Message: "update failed", QueuedAt: now.Add(-58 * time.Minute), + }) + if err != nil || !queued { + t.Fatalf("queue=%v err=%v", queued, err) + } + notifier := &recordingNotifier{} + result, err := (Service{Database: database, Notifier: notifier, Enabled: true, Now: func() time.Time { return now }}).Check(ctx, 2*time.Hour) + if err != nil || result.Healthy || result.Alerted || result.DesktopNotified || notifier.calls != 0 { + t.Fatalf("result=%#v calls=%d err=%v", result, notifier.calls, err) + } +} From 8083706633a11203081eb05247ca72acd6973980 Mon Sep 17 00:00:00 2001 From: z2z23n0 Date: Sat, 18 Jul 2026 12:03:32 +0800 Subject: [PATCH 2/5] test: cover exact git resolver refs --- internal/bundle/resolver_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 internal/bundle/resolver_test.go diff --git a/internal/bundle/resolver_test.go b/internal/bundle/resolver_test.go new file mode 100644 index 0000000..555efb9 --- /dev/null +++ b/internal/bundle/resolver_test.go @@ -0,0 +1,30 @@ +package bundle + +import ( + "context" + "testing" + + "github.com/z2z23n0/tooltend/internal/execx" + "github.com/z2z23n0/tooltend/internal/model" +) + +type resolverRunner struct { + stdout []byte +} + +func (r resolverRunner) Run(context.Context, string, ...string) (execx.Result, error) { + return execx.Result{Stdout: r.stdout}, nil +} + +func TestRunResolverAcceptsExactGitCommit(t *testing.T) { + const resolved = "git:ABCDEF0123456789ABCDEF0123456789ABCDEF01" + service := Service{Runner: resolverRunner{stdout: []byte(resolved + "\n")}} + + got, err := service.runResolver(context.Background(), []string{"resolver"}, model.Installation{}) + if err != nil { + t.Fatal(err) + } + if want := "git:abcdef0123456789abcdef0123456789abcdef01"; got != want { + t.Fatalf("resolved version = %q, want %q", got, want) + } +} From ca204f295ed41243fa0301339f12acb162985d15 Mon Sep 17 00:00:00 2001 From: z2z23n0 Date: Wed, 15 Jul 2026 22:52:39 +0800 Subject: [PATCH 3/5] feat: manage selected bundle lifecycles --- internal/bundle/discover.go | 53 ++ internal/bundle/discover_test.go | 33 + internal/bundle/recipe.go | 8 + internal/bundle/recipes/agent-capsule.toml | 15 +- internal/bundle/recipes/anysearch.toml | 9 +- internal/bundle/recipes/codex-conductor.toml | 9 +- internal/bundle/recipes/known-skills.toml | 9 +- internal/bundle/recipes/mainline.toml | 20 +- internal/bundle/recipes/sherlog.toml | 15 +- internal/bundle/recipes/shuorenhua.toml | 9 +- internal/bundle/recipes/xsearch.toml | 22 +- internal/bundle/service.go | 124 ++- internal/bundle/service_test.go | 44 + internal/bundledriver/driver.go | 913 +++++++++++++++++++ internal/bundledriver/driver_test.go | 145 +++ internal/cli/app.go | 4 + internal/cli/bundle_commands.go | 4 +- internal/cli/bundle_driver_commands.go | 24 + internal/cli/worker_commands.go | 3 + internal/store/bundles.go | 4 + internal/store/bundles_test.go | 35 + 21 files changed, 1475 insertions(+), 27 deletions(-) create mode 100644 internal/bundledriver/driver.go create mode 100644 internal/bundledriver/driver_test.go create mode 100644 internal/cli/bundle_driver_commands.go diff --git a/internal/bundle/discover.go b/internal/bundle/discover.go index a8ba491..c35bd73 100644 --- a/internal/bundle/discover.go +++ b/internal/bundle/discover.go @@ -97,6 +97,7 @@ func Discover(ctx context.Context, database *store.Store, options DiscoverOption match := matchedFromObserved(*item, artifact, options.HomeDir) enrichSkillMatch(&match, *item, skillEvidence) enrichWorkspaceMatch(&match, *item) + enrichRecipeSource(&match, artifact) matches[artifact.Key] = append(matches[artifact.Key], match) matchedBindings[item.binding.ID] = struct{}{} } @@ -104,9 +105,17 @@ func Discover(ctx context.Context, database *store.Store, options DiscoverOption for _, probe := range artifact.Probes { match, ok := resolveProbe(ctx, probe, recipe, artifact, options, lookup) if ok { + enrichRecipeSource(&match, artifact) matches[artifact.Key] = append(matches[artifact.Key], match) } } + if artifact.Driver == "mainline-hooks" { + hookMatches, hookErr := discoverMainlineHooks(ctx, database) + if hookErr != nil { + return DiscoverResult{}, hookErr + } + matches[artifact.Key] = append(matches[artifact.Key], hookMatches...) + } matches[artifact.Key] = dedupeMatches(matches[artifact.Key]) } if countMatches(matches) == 0 { @@ -588,6 +597,50 @@ func enrichWorkspaceMatch(match *matchedInstallation, observed observedInstallat } } +func enrichRecipeSource(match *matchedInstallation, artifact ArtifactRecipe) { + if strings.TrimSpace(artifact.Source) == "" { + return + } + match.sourceIdentity = strings.TrimSpace(artifact.Source) + if artifact.Subdir != "" { + match.sourceIdentity += "#" + filepath.ToSlash(filepath.Clean(artifact.Subdir)) + } + match.metadata["recipe_source_url"] = strings.TrimSpace(artifact.Source) + if artifact.Subdir != "" { + match.metadata["recipe_source_subdir"] = filepath.ToSlash(filepath.Clean(artifact.Subdir)) + } +} + +func discoverMainlineHooks(ctx context.Context, database *store.Store) ([]matchedInstallation, error) { + projects, err := database.ListProjects(ctx) + if err != nil { + return nil, err + } + var result []matchedInstallation + for _, project := range projects { + if !project.Selected { + continue + } + root := filepath.Clean(project.RootPath) + if info, statErr := os.Stat(filepath.Join(root, ".mainline", "config.toml")); statErr != nil || !info.Mode().IsRegular() { + continue + } + digest := sha256.New() + for _, relative := range []string{".claude/settings.json", ".codex/config.toml", ".codex/hooks.json", ".cursor/hooks.json"} { + data, readErr := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative))) + if readErr == nil { + _, _ = digest.Write([]byte(relative + "\x00")) + _, _ = digest.Write(data) + } + } + result = append(result, matchedInstallation{ + path: root, packageIdentity: "mainline-hooks", sourceIdentity: "mainline-hooks:" + root, + hash: hex.EncodeToString(digest.Sum(nil)), metadata: map[string]any{"project_id": project.ID, "derived": true}, + }) + } + return result, nil +} + func inspectSignedSkillManifest(skillPath string) (string, bool, bool) { manifestPath := filepath.Join(skillPath, "skill.manifest") data, err := os.ReadFile(manifestPath) diff --git a/internal/bundle/discover_test.go b/internal/bundle/discover_test.go index 93b6f56..bc9fd84 100644 --- a/internal/bundle/discover_test.go +++ b/internal/bundle/discover_test.go @@ -88,6 +88,39 @@ func TestDiscoverDeduplicatesPhysicalInstallAndReadsPackageMetadata(t *testing.T } } +func TestDiscoverMainlineHooksUsesSelectedRepositories(t *testing.T) { + database, err := store.OpenRW(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + ctx := context.Background() + selected := t.TempDir() + ignored := t.TempDir() + for _, root := range []string{selected, ignored} { + if err := os.MkdirAll(filepath.Join(root, ".mainline"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".mainline", "config.toml"), []byte("[hooks]\nenabled=true\n"), 0o644); err != nil { + t.Fatal(err) + } + } + now := time.Now().UTC() + if err := database.UpsertProject(ctx, model.Project{ID: "selected", RootPath: selected, RootFingerprint: "selected", Selected: true, DiscoveredVia: "test", LastSeenAt: now}); err != nil { + t.Fatal(err) + } + if err := database.UpsertProject(ctx, model.Project{ID: "ignored", RootPath: ignored, RootFingerprint: "ignored", Selected: false, DiscoveredVia: "test", LastSeenAt: now}); err != nil { + t.Fatal(err) + } + matches, err := discoverMainlineHooks(ctx, database) + if err != nil { + t.Fatal(err) + } + if len(matches) != 1 || matches[0].path != selected || matches[0].packageIdentity != "mainline-hooks" { + t.Fatalf("matches = %#v", matches) + } +} + func TestDiscoverPrunesStaleProbeWhenBindingProvidesRicherEvidence(t *testing.T) { database, err := store.OpenRW(filepath.Join(t.TempDir(), "state.db")) if err != nil { diff --git a/internal/bundle/recipe.go b/internal/bundle/recipe.go index 84b8916..58d6f37 100644 --- a/internal/bundle/recipe.go +++ b/internal/bundle/recipe.go @@ -45,6 +45,8 @@ type ArtifactRecipe struct { Name string `toml:"name" json:"name"` Kind model.ArtifactKind `toml:"kind" json:"kind"` Driver string `toml:"driver" json:"driver"` + Source string `toml:"source" json:"source,omitempty"` + Subdir string `toml:"subdir" json:"subdir,omitempty"` Required bool `toml:"required" json:"required"` Selectors []Selector `toml:"selectors" json:"selectors"` Probes []string `toml:"probes" json:"probes,omitempty"` @@ -184,6 +186,12 @@ func (r Recipe) Validate() error { if err := artifact.Kind.Validate(); err != nil { return err } + if strings.ContainsAny(artifact.Source+artifact.Subdir, "\x00\r\n") { + return fmt.Errorf("artifact %s source contains invalid characters", artifact.Key) + } + if artifact.Subdir != "" && (filepath.IsAbs(artifact.Subdir) || strings.HasPrefix(filepath.Clean(artifact.Subdir), "..")) { + return fmt.Errorf("artifact %s source subdirectory must be relative", artifact.Key) + } for _, selector := range artifact.Selectors { if err := selector.Validate(); err != nil { return fmt.Errorf("artifact %s: %w", artifact.Key, err) diff --git a/internal/bundle/recipes/agent-capsule.toml b/internal/bundle/recipes/agent-capsule.toml index 3eb6bae..cc8c42a 100644 --- a/internal/bundle/recipes/agent-capsule.toml +++ b/internal/bundle/recipes/agent-capsule.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "agent-capsule" -version = "1" +version = "2" name = "Agent Capsule" owner = "delegated" confidence = "high" @@ -13,7 +13,11 @@ kind = "cli" driver = "github-release" required = true probes = ["command:capsule"] -health_argv = ["capsule", "version"] +resolve_argv = ["tooltend", "__bundle-driver", "github-resolve", "z2z23n0/agent-capsule"] +stage_argv = ["tooltend", "__bundle-driver", "github-stage", "z2z23n0/agent-capsule", "capsule", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "github-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "github-rollback", "z2z23n0/agent-capsule", "capsule", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "binary-health", "${path}", "help"] [[artifacts.selectors]] field = "name" equals = "capsule" @@ -23,7 +27,14 @@ key = "skill" name = "Agent Capsule Skill" kind = "skill" driver = "git-skill" +source = "https://github.com/z2z23n0/agent-capsule.git" +subdir = "skills/agent-capsule" required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-release-resolve", "https://github.com/z2z23n0/agent-capsule.git", "z2z23n0/agent-capsule"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/z2z23n0/agent-capsule.git", "skills/agent-capsule", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/z2z23n0/agent-capsule.git", "skills/agent-capsule", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "agent-capsule" diff --git a/internal/bundle/recipes/anysearch.toml b/internal/bundle/recipes/anysearch.toml index f58fe34..1bea9a9 100644 --- a/internal/bundle/recipes/anysearch.toml +++ b/internal/bundle/recipes/anysearch.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "anysearch" -version = "1" +version = "2" name = "AnySearch" owner = "delegated" confidence = "high" @@ -9,7 +9,14 @@ key = "skill" name = "AnySearch Skill" kind = "skill" driver = "git-skill" +source = "https://github.com/catoncat/anysearch-skill.git" +subdir = "anysearch" required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-resolve", "https://github.com/catoncat/anysearch-skill.git"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/catoncat/anysearch-skill.git", "anysearch", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/catoncat/anysearch-skill.git", "anysearch", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "anysearch" diff --git a/internal/bundle/recipes/codex-conductor.toml b/internal/bundle/recipes/codex-conductor.toml index 71222ae..17e44bc 100644 --- a/internal/bundle/recipes/codex-conductor.toml +++ b/internal/bundle/recipes/codex-conductor.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "codex-conductor" -version = "1" +version = "2" name = "Codex Conductor" owner = "delegated" confidence = "high" @@ -9,7 +9,14 @@ key = "skill" name = "Codex Conductor Skill" kind = "skill" driver = "git-skill" +source = "https://github.com/catoncat/codex-conductor.git" +subdir = "." required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-resolve", "https://github.com/catoncat/codex-conductor.git"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/catoncat/codex-conductor.git", ".", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/catoncat/codex-conductor.git", ".", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "codex-conductor" diff --git a/internal/bundle/recipes/known-skills.toml b/internal/bundle/recipes/known-skills.toml index 3037dba..7864d59 100644 --- a/internal/bundle/recipes/known-skills.toml +++ b/internal/bundle/recipes/known-skills.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "0g-hk" -version = "1" +version = "2" name = "0g-hk" owner = "delegated" confidence = "high" @@ -10,7 +10,14 @@ key = "skill" name = "0g-hk Skill" kind = "skill" driver = "git-skill" +source = "https://github.com/catoncat/0g-hk.git" +subdir = "skill-packages/0g-hk" required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-resolve", "https://github.com/catoncat/0g-hk.git"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/catoncat/0g-hk.git", "skill-packages/0g-hk", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/catoncat/0g-hk.git", "skill-packages/0g-hk", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "0g-hk" diff --git a/internal/bundle/recipes/mainline.toml b/internal/bundle/recipes/mainline.toml index d6542dd..d01c1d0 100644 --- a/internal/bundle/recipes/mainline.toml +++ b/internal/bundle/recipes/mainline.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "mainline" -version = "1" +version = "2" name = "Mainline" owner = "delegated" confidence = "high" @@ -13,7 +13,11 @@ kind = "cli" driver = "github-release" required = true probes = ["command:mainline"] -health_argv = ["mainline", "version"] +resolve_argv = ["tooltend", "__bundle-driver", "github-resolve", "mainline-org/mainline"] +stage_argv = ["tooltend", "__bundle-driver", "github-stage", "mainline-org/mainline", "mainline", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "github-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "github-rollback", "mainline-org/mainline", "mainline", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "binary-health", "${path}", "version"] [[artifacts.selectors]] field = "name" equals = "mainline" @@ -26,7 +30,14 @@ key = "skill" name = "Mainline Skill" kind = "skill" driver = "npx-skills" +source = "https://github.com/mainline-org/mainline.git" +subdir = "skills/mainline" required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-release-resolve", "https://github.com/mainline-org/mainline.git", "mainline-org/mainline"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/mainline-org/mainline.git", "skills/mainline", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/mainline-org/mainline.git", "skills/mainline", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "mainline" @@ -40,3 +51,8 @@ name = "Mainline generated hooks" kind = "hook" driver = "mainline-hooks" required = false +resolve_argv = ["tooltend", "__bundle-driver", "github-resolve", "mainline-org/mainline"] +stage_argv = ["tooltend", "__bundle-driver", "mainline-hooks-stage", "${path}", "${stage}"] +activate_argv = ["tooltend", "__bundle-driver", "mainline-hooks-activate", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "mainline-hooks-rollback", "${path}", "${stage}"] +health_argv = ["tooltend", "__bundle-driver", "mainline-hooks-health", "${path}"] diff --git a/internal/bundle/recipes/sherlog.toml b/internal/bundle/recipes/sherlog.toml index 525938a..eae0b01 100644 --- a/internal/bundle/recipes/sherlog.toml +++ b/internal/bundle/recipes/sherlog.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "sherlog" -version = "1" +version = "2" name = "Sherlog" owner = "delegated" confidence = "high" @@ -13,7 +13,11 @@ kind = "cli" driver = "npm" required = true probes = ["command:sherlog", "command:shlog"] -health_argv = ["sherlog", "--version"] +resolve_argv = ["tooltend", "__bundle-driver", "npm-resolve", "@act0r/sherlog"] +stage_argv = ["tooltend", "__bundle-driver", "npm-stage", "@act0r/sherlog", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "npm-activate", "${stage}"] +rollback_argv = ["tooltend", "__bundle-driver", "npm-rollback", "@act0r/sherlog", "${rollback_version}", "${stage}"] +health_argv = ["shlog", "--version"] [[artifacts.selectors]] field = "name" equals = "sherlog|shlog" @@ -26,7 +30,14 @@ key = "skill" name = "Sherlog Skill" kind = "skill" driver = "npx-skills" +source = "https://github.com/catoncat/sherlog.git" +subdir = "skill-packages/sherlog" required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-npm-release-resolve", "https://github.com/catoncat/sherlog.git", "@act0r/sherlog"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/catoncat/sherlog.git", "skill-packages/sherlog", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/catoncat/sherlog.git", "skill-packages/sherlog", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "sherlog" diff --git a/internal/bundle/recipes/shuorenhua.toml b/internal/bundle/recipes/shuorenhua.toml index 5628b63..4998c40 100644 --- a/internal/bundle/recipes/shuorenhua.toml +++ b/internal/bundle/recipes/shuorenhua.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "shuorenhua" -version = "1" +version = "2" name = "shuorenhua" owner = "delegated" confidence = "high" @@ -9,7 +9,14 @@ key = "skill" name = "shuorenhua Skill" kind = "skill" driver = "git-skill" +source = "https://github.com/MrGeDiao/shuorenhua.git" +subdir = "." required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-resolve", "https://github.com/MrGeDiao/shuorenhua.git"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/MrGeDiao/shuorenhua.git", ".", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/MrGeDiao/shuorenhua.git", ".", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "shuorenhua" diff --git a/internal/bundle/recipes/xsearch.toml b/internal/bundle/recipes/xsearch.toml index ddf05a6..4b4a378 100644 --- a/internal/bundle/recipes/xsearch.toml +++ b/internal/bundle/recipes/xsearch.toml @@ -1,6 +1,6 @@ schema = "bundle-recipe-v1" id = "xsearch" -version = "1" +version = "2" name = "xsearch" owner = "delegated" confidence = "high" @@ -9,7 +9,27 @@ key = "skill" name = "xsearch Skill" kind = "skill" driver = "git-skill" +source = "https://github.com/catoncat/xsearch.git" +subdir = "." required = true +resolve_argv = ["tooltend", "__bundle-driver", "git-resolve", "https://github.com/catoncat/xsearch.git"] +stage_argv = ["tooltend", "__bundle-driver", "git-stage", "https://github.com/catoncat/xsearch.git", ".", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "git-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "git-rollback", "https://github.com/catoncat/xsearch.git", ".", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "skill-health", "${path}"] [[artifacts.selectors]] field = "name" equals = "xsearch" + +[[artifacts]] +key = "binary" +name = "xsearch embedded binary" +kind = "embedded_binary" +driver = "github-release" +required = true +probes = ["path:~/.agents/skills/xsearch/bin/xsearch"] +resolve_argv = ["tooltend", "__bundle-driver", "github-resolve", "catoncat/xsearch"] +stage_argv = ["tooltend", "__bundle-driver", "github-stage", "catoncat/xsearch", "xsearch", "${version}", "${previous_version}", "${stage}", "${path}"] +activate_argv = ["tooltend", "__bundle-driver", "github-activate", "${stage}", "${path}"] +rollback_argv = ["tooltend", "__bundle-driver", "github-rollback", "catoncat/xsearch", "xsearch", "${rollback_version}", "${stage}", "${path}"] +health_argv = ["tooltend", "__bundle-driver", "binary-health", "${path}", "--version"] diff --git a/internal/bundle/service.go b/internal/bundle/service.go index 02bef97..4784ff9 100644 --- a/internal/bundle/service.go +++ b/internal/bundle/service.go @@ -12,6 +12,8 @@ import ( "strings" "time" + semver "github.com/Masterminds/semver/v3" + "github.com/z2z23n0/tooltend/internal/config" "github.com/z2z23n0/tooltend/internal/execx" "github.com/z2z23n0/tooltend/internal/model" @@ -26,19 +28,21 @@ type Service struct { } type UpdatePreview struct { - Bundle model.Bundle `json:"bundle"` - Policy model.BundlePolicy `json:"policy"` - Current *model.BundleRelease `json:"current_release,omitempty"` - Target model.BundleRelease `json:"target_release"` - Artifacts []UpdateArtifactPreview `json:"artifacts"` - StageOnly bool `json:"stage_only"` - AutoEligible bool `json:"auto_eligible"` + Bundle model.Bundle `json:"bundle"` + Policy model.BundlePolicy `json:"policy"` + Current *model.BundleRelease `json:"current_release,omitempty"` + Target model.BundleRelease `json:"target_release"` + Artifacts []UpdateArtifactPreview `json:"artifacts"` + StageOnly bool `json:"stage_only"` + AutoEligible bool `json:"auto_eligible"` + UpdateAvailable bool `json:"update_available"` } type UpdateArtifactPreview struct { Artifact model.BundleArtifact `json:"artifact"` Installations int `json:"installations"` ResolvedVersion string `json:"resolved_version,omitempty"` + Changed bool `json:"changed"` CanStage bool `json:"can_stage"` CanActivate bool `json:"can_activate"` CanRollback bool `json:"can_rollback"` @@ -108,6 +112,13 @@ func (s Service) PrepareUpdate(ctx context.Context, bundleID string, stageOnly b preview.Current = ¤t } } + currentVersions := map[string]string{} + if preview.Current != nil { + currentVersions, err = parseReleaseVersions(preview.Current.ManifestJSON) + if err != nil { + return UpdatePreview{}, fmt.Errorf("current bundle release manifest: %w", err) + } + } versions := map[string]string{} for _, artifact := range artifacts { recipe, err := decodeArtifactMetadata(artifact) @@ -133,7 +144,12 @@ func (s Service) PrepareUpdate(ctx context.Context, bundleID string, stageOnly b if err != nil { return UpdatePreview{}, fmt.Errorf("resolve artifact %s: %w", artifact.Name, err) } + currentVersion := currentVersions[artifact.RecipeKey] + if compareArtifactVersions(resolved, currentVersion) < 0 { + resolved = currentVersion + } item.ResolvedVersion = resolved + item.Changed = resolved != currentVersion versions[artifact.RecipeKey] = resolved if !item.CanStage || !item.CanActivate || !item.CanRollback || !item.CanHealthCheck { preview.AutoEligible = false @@ -158,6 +174,7 @@ func (s Service) PrepareUpdate(ctx context.Context, bundleID string, stageOnly b ID: stableID("rel", bundleValue.ID+"\x00"+string(manifest)), BundleID: bundleValue.ID, Version: releaseVersion, ResolvedRef: releaseVersion, ManifestJSON: string(manifest), Status: "resolved", CreatedAt: s.now(), } + preview.UpdateAvailable = preview.Current == nil || !artifactVersionMapsEqual(currentVersions, versions) return preview, nil } @@ -165,6 +182,9 @@ func (s Service) ExecuteUpdate(ctx context.Context, preview UpdatePreview) (resu if err := s.validate(); err != nil { return result, err } + if !preview.UpdateAvailable { + return result, errors.New("bundle is already at the resolved release") + } currentBundle, err := s.Database.GetBundle(ctx, preview.Bundle.ID) if err != nil { return result, err @@ -216,6 +236,10 @@ func (s Service) ExecuteUpdate(ctx context.Context, preview UpdatePreview) (resu byArtifact[installation.ArtifactID] = append(byArtifact[installation.ArtifactID], installation) } versions := releaseVersions(preview.Target.ManifestJSON) + currentVersions := map[string]string{} + if preview.Current != nil { + currentVersions = releaseVersions(preview.Current.ManifestJSON) + } steps := []executionStep{} ordinal := 0 for _, artifact := range artifacts { @@ -227,6 +251,9 @@ func (s Service) ExecuteUpdate(ctx context.Context, preview UpdatePreview) (resu if len(recipe.StageArgv) == 0 && len(recipe.ActivateArgv) == 0 { continue } + if versions[artifact.RecipeKey] == currentVersions[artifact.RecipeKey] { + continue + } stepID := stableID("bst", transactionID+fmt.Sprintf("\x00%d", ordinal)) step := executionStep{ record: model.BundleTransactionStep{ID: stepID, TransactionID: transactionID, Ordinal: ordinal, ArtifactID: artifact.ID, @@ -410,7 +437,7 @@ func (s Service) PrepareRollback(ctx context.Context, bundleID, targetReleaseID continue } version := targetVersions[artifact.RecipeKey] - if count > 0 && !exactVersion(version) { + if count > 0 && !exactArtifactVersion(version) { return RollbackPreview{}, fmt.Errorf("rollback target has no exact version for artifact %s", artifact.Name) } recipe, err := decodeArtifactMetadata(artifact) @@ -501,7 +528,7 @@ func (s Service) ExecuteRollback(ctx context.Context, preview RollbackPreview) ( } transaction.Status = model.BundleTransactionRollingBack completedSteps := make([]executionStep, 0, len(steps)) - for index := len(steps) - 1; index >= 0; index-- { + for _, index := range explicitRollbackOrder(steps) { step := steps[index] if err := s.Database.UpdateBundleTransactionStep(ctx, step.record.ID, model.BundleStepCompensating, "", "", "{}", nil); err != nil { return result, s.failTransaction(ctx, transaction, "journal_failed", err) @@ -551,14 +578,15 @@ func (s Service) ExecuteRollback(ctx context.Context, preview RollbackPreview) ( func (s Service) restoreAfterRollbackFailure(ctx context.Context, completed []executionStep, versions map[string]string) error { var failures []error - for index := len(completed) - 1; index >= 0; index-- { + for _, index := range explicitRollbackOrder(completed) { step := completed[index] - step.version = versions[step.artifact.RecipeKey] - if !exactVersion(step.version) { + step.rollbackVersion = versions[step.artifact.RecipeKey] + step.version = step.rollbackVersion + if !exactArtifactVersion(step.rollbackVersion) { failures = append(failures, fmt.Errorf("artifact %s has no exact restore version", step.artifact.Name)) continue } - if err := s.runCommand(context.WithoutCancel(ctx), step.recipe.ActivateArgv, step, DefaultInstallTimeout); err != nil { + if err := s.runCommand(context.WithoutCancel(ctx), step.recipe.RollbackArgv, step, DefaultInstallTimeout); err != nil { failures = append(failures, err) } } @@ -585,8 +613,11 @@ func (s Service) runResolver(ctx context.Context, argv []string, installation mo if index := strings.IndexByte(version, '\n'); index >= 0 { version = strings.TrimSpace(version[:index]) } - if !exactVersion(version) { - return "", errors.New("resolver did not return an exact semantic version") + if !exactArtifactVersion(version) { + return "", errors.New("resolver did not return an exact semantic version or git commit") + } + if strings.HasPrefix(version, "git:") { + return strings.ToLower(version), nil } return strings.TrimPrefix(version, "v"), nil } @@ -783,3 +814,66 @@ func bundleReleaseVersion(versions map[string]string, manifest []byte) string { } return "bundle-" + strings.TrimPrefix(stableID("", string(manifest)), "_")[:12] } + +func exactArtifactVersion(value string) bool { + value = strings.TrimSpace(value) + if strings.HasPrefix(value, "git:") { + commit := strings.TrimPrefix(value, "git:") + if len(commit) != 40 { + return false + } + for _, character := range commit { + if !strings.ContainsRune("0123456789abcdefABCDEF", character) { + return false + } + } + return true + } + return exactVersion(value) +} + +func compareArtifactVersions(candidate, current string) int { + candidate, current = strings.TrimSpace(candidate), strings.TrimSpace(current) + if current == "" { + return 1 + } + if candidate == current { + return 0 + } + if strings.HasPrefix(candidate, "git:") || strings.HasPrefix(current, "git:") { + return 1 + } + candidateVersion, candidateErr := semver.NewVersion(strings.TrimPrefix(candidate, "v")) + currentVersion, currentErr := semver.NewVersion(strings.TrimPrefix(current, "v")) + if candidateErr != nil || currentErr != nil { + return 1 + } + return candidateVersion.Compare(currentVersion) +} + +func artifactVersionMapsEqual(left, right map[string]string) bool { + if len(left) != len(right) { + return false + } + for key, value := range left { + if right[key] != value { + return false + } + } + return true +} + +func explicitRollbackOrder(steps []executionStep) []int { + order := make([]int, 0, len(steps)) + for index := len(steps) - 1; index >= 0; index-- { + if steps[index].artifact.Kind != model.ArtifactHook { + order = append(order, index) + } + } + for index := len(steps) - 1; index >= 0; index-- { + if steps[index].artifact.Kind == model.ArtifactHook { + order = append(order, index) + } + } + return order +} diff --git a/internal/bundle/service_test.go b/internal/bundle/service_test.go index c454150..de59adf 100644 --- a/internal/bundle/service_test.go +++ b/internal/bundle/service_test.go @@ -28,6 +28,8 @@ func (r *transactionRunner) Run(_ context.Context, name string, args ...string) switch name { case "resolver": return execx.Result{Stdout: []byte("2.0.0\n")}, nil + case "git-resolver": + return execx.Result{Stdout: []byte("git:0123456789abcdef0123456789abcdef01234567\n")}, nil case "activate-two": return execx.Result{}, errors.New("activation failed") default: @@ -35,6 +37,48 @@ func (r *transactionRunner) Run(_ context.Context, name string, args ...string) } } +func TestArtifactVersionComparisonAndRollbackOrder(t *testing.T) { + if !exactArtifactVersion("git:0123456789abcdef0123456789abcdef01234567") { + t.Fatal("exact git commit was rejected") + } + if exactArtifactVersion("git:main") { + t.Fatal("symbolic git ref was accepted") + } + if compareArtifactVersions("1.2.3", "1.2.4") >= 0 { + t.Fatal("semantic downgrade was not detected") + } + if !artifactVersionMapsEqual(map[string]string{"cli": "1.2.3"}, map[string]string{"cli": "1.2.3"}) { + t.Fatal("equivalent observed and resolved manifests were treated as an update") + } + steps := []executionStep{ + {artifact: model.BundleArtifact{Kind: model.ArtifactCLI}}, + {artifact: model.BundleArtifact{Kind: model.ArtifactSkill}}, + {artifact: model.BundleArtifact{Kind: model.ArtifactHook}}, + } + order := explicitRollbackOrder(steps) + if len(order) != 3 || order[0] != 1 || order[1] != 0 || order[2] != 2 { + t.Fatalf("rollback order = %v", order) + } +} + +func TestSelectedBuiltinRecipesAreAutoCapable(t *testing.T) { + catalog, err := LoadCatalog("") + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"sherlog", "mainline", "agent-capsule", "0g-hk", "anysearch", "codex-conductor", "shuorenhua", "xsearch"} { + recipe, ok := catalog.Get(id) + if !ok { + t.Fatalf("recipe %s is missing", id) + } + for _, artifact := range recipe.Artifacts { + if len(artifact.ResolveArgv) == 0 || len(artifact.StageArgv) == 0 || len(artifact.ActivateArgv) == 0 || len(artifact.RollbackArgv) == 0 || len(artifact.HealthArgv) == 0 { + t.Fatalf("recipe %s artifact %s is not auto capable", id, artifact.Key) + } + } + } +} + func TestBundleTransactionStagesAllArtifactsBeforeActivationAndCompensates(t *testing.T) { root := t.TempDir() paths := config.ResolveWith(root, func(key string) string { diff --git a/internal/bundledriver/driver.go b/internal/bundledriver/driver.go new file mode 100644 index 0000000..54a63a1 --- /dev/null +++ b/internal/bundledriver/driver.go @@ -0,0 +1,913 @@ +package bundledriver + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + semver "github.com/Masterminds/semver/v3" + + "github.com/z2z23n0/tooltend/internal/execx" + "github.com/z2z23n0/tooltend/internal/safeio" +) + +const maxDownloadBytes = 256 << 20 + +type Driver struct { + Runner execx.Runner + Client *http.Client + Out io.Writer + GOOS string + GOARCH string +} + +func (d Driver) Execute(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("bundle driver action is required") + } + switch args[0] { + case "npm-resolve": + if len(args) != 2 { + return errors.New("npm-resolve requires package") + } + return d.npmResolve(ctx, args[1]) + case "npm-stage": + if len(args) != 6 { + return errors.New("npm-stage requires package, version, previous version, stage, and path") + } + return d.npmStage(ctx, args[1], args[2], args[3], args[4], args[5]) + case "npm-activate": + if len(args) != 2 { + return errors.New("npm-activate requires stage") + } + return d.npmInstallArchive(ctx, filepath.Join(args[1], "target.tgz")) + case "npm-rollback": + if len(args) != 4 { + return errors.New("npm-rollback requires package, version, and stage") + } + return d.npmRollback(ctx, args[1], args[2], args[3]) + case "github-resolve": + if len(args) != 2 { + return errors.New("github-resolve requires repository") + } + return d.githubResolve(ctx, args[1]) + case "github-stage": + if len(args) != 7 { + return errors.New("github-stage requires repository, binary, version, previous version, stage, and path") + } + return d.githubStage(ctx, args[1], args[2], args[3], args[4], args[5], args[6]) + case "github-activate": + if len(args) != 3 { + return errors.New("github-activate requires stage and path") + } + return replacePath(filepath.Join(args[1], "next"), args[2]) + case "github-rollback": + if len(args) != 6 { + return errors.New("github-rollback requires repository, binary, version, stage, and path") + } + return d.githubRollback(ctx, args[1], args[2], args[3], args[4], args[5]) + case "git-resolve": + if len(args) != 2 { + return errors.New("git-resolve requires repository URL") + } + return d.gitResolve(ctx, args[1]) + case "git-release-resolve": + if len(args) != 3 { + return errors.New("git-release-resolve requires repository URL and GitHub repository") + } + return d.gitReleaseResolve(ctx, args[1], args[2]) + case "git-npm-release-resolve": + if len(args) != 3 { + return errors.New("git-npm-release-resolve requires repository URL and npm package") + } + return d.gitNPMReleaseResolve(ctx, args[1], args[2]) + case "git-stage": + if len(args) != 7 { + return errors.New("git-stage requires repository, subdirectory, ref, previous ref, stage, and path") + } + return d.gitStage(ctx, args[1], args[2], args[3], args[4], args[5], args[6]) + case "git-activate": + if len(args) != 3 { + return errors.New("git-activate requires stage and path") + } + if err := replacePath(filepath.Join(args[1], "next"), args[2]); err != nil { + return err + } + return detachSkillLock(args[2]) + case "git-rollback": + if len(args) != 6 { + return errors.New("git-rollback requires repository, subdirectory, ref, stage, and path") + } + return d.gitRollback(ctx, args[1], args[2], args[3], args[4], args[5]) + case "skill-health": + if len(args) != 2 { + return errors.New("skill-health requires path") + } + return skillHealth(args[1]) + case "binary-health": + if len(args) < 2 { + return errors.New("binary-health requires path") + } + _, err := d.runner().Run(ctx, args[1], args[2:]...) + if err != nil { + return errors.New("managed binary health check failed") + } + return nil + case "mainline-hooks-stage": + if len(args) != 3 { + return errors.New("mainline-hooks-stage requires project and stage") + } + return stageMainlineHooks(args[1], args[2]) + case "mainline-hooks-activate": + if len(args) != 2 { + return errors.New("mainline-hooks-activate requires project") + } + return d.runMainlineHooks(ctx, args[1], "install") + case "mainline-hooks-rollback": + if len(args) != 3 { + return errors.New("mainline-hooks-rollback requires project and stage") + } + if restored, err := restoreMainlineHooks(args[1], args[2]); err != nil || restored { + return err + } + return d.runMainlineHooks(ctx, args[1], "install") + case "mainline-hooks-health": + if len(args) != 2 { + return errors.New("mainline-hooks-health requires project") + } + return d.runMainlineHooks(ctx, args[1], "status") + default: + return fmt.Errorf("unsupported bundle driver action %q", args[0]) + } +} + +func (d Driver) runner() execx.Runner { + if d.Runner != nil { + return d.Runner + } + return execx.ExecRunner{} +} + +func (d Driver) output(value string) error { + w := d.Out + if w == nil { + w = os.Stdout + } + _, err := fmt.Fprintln(w, value) + return err +} + +func (d Driver) npmResolve(ctx context.Context, packageName string) error { + version, err := d.npmVersion(ctx, packageName) + if err != nil { + return err + } + return d.output(version) +} + +func (d Driver) npmVersion(ctx context.Context, packageName string) (string, error) { + result, err := d.runner().Run(ctx, "npm", "view", packageName, "version", "--json") + if err != nil { + return "", errors.New("npm version lookup failed") + } + var version string + if json.Unmarshal(result.Stdout, &version) != nil { + var versions []string + if json.Unmarshal(result.Stdout, &versions) != nil || len(versions) == 0 { + return "", errors.New("npm returned an invalid version") + } + version = versions[len(versions)-1] + } + if _, err := semver.StrictNewVersion(version); err != nil { + return "", errors.New("npm returned an invalid semantic version") + } + return version, nil +} + +func (d Driver) npmStage(ctx context.Context, packageName, version, previous, stage, path string) error { + if _, err := semver.StrictNewVersion(version); err != nil { + return errors.New("npm target version is invalid") + } + if err := resetStage(stage); err != nil { + return err + } + if err := d.npmPack(ctx, packageName, version, stage, "target.tgz"); err != nil { + return err + } + if _, err := semver.StrictNewVersion(strings.TrimPrefix(previous, "v")); err == nil { + if err := d.npmPack(ctx, packageName, strings.TrimPrefix(previous, "v"), stage, "previous.tgz"); err != nil { + return err + } + } + return backupPath(path, filepath.Join(stage, "previous-installation")) +} + +func (d Driver) npmPack(ctx context.Context, packageName, version, stage, target string) error { + result, err := d.runner().Run(ctx, "npm", "pack", packageName+"@"+version, "--json", "--pack-destination", stage) + if err != nil { + return errors.New("npm package staging failed") + } + var records []struct { + Filename string `json:"filename"` + } + if json.Unmarshal(result.Stdout, &records) != nil || len(records) != 1 || filepath.Base(records[0].Filename) != records[0].Filename { + return errors.New("npm package staging returned invalid metadata") + } + return os.Rename(filepath.Join(stage, records[0].Filename), filepath.Join(stage, target)) +} + +func (d Driver) npmInstallArchive(ctx context.Context, archive string) error { + if info, err := os.Stat(archive); err != nil || !info.Mode().IsRegular() { + return errors.New("staged npm package is missing") + } + if _, err := d.runner().Run(ctx, "npm", "install", "--global", "--no-audit", "--no-fund", archive); err != nil { + return errors.New("npm package activation failed") + } + return nil +} + +func (d Driver) npmRollback(ctx context.Context, packageName, version, stage string) error { + archive := filepath.Join(stage, "previous.tgz") + if info, err := os.Stat(archive); err == nil && info.Mode().IsRegular() { + return d.npmInstallArchive(ctx, archive) + } + version = strings.TrimPrefix(version, "v") + if _, err := semver.StrictNewVersion(version); err != nil { + return errors.New("npm rollback version is unavailable") + } + if _, err := d.runner().Run(ctx, "npm", "install", "--global", "--no-audit", "--no-fund", packageName+"@"+version); err != nil { + return errors.New("npm package rollback failed") + } + return nil +} + +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []githubAsset `json:"assets"` +} + +type githubAsset struct { + Name string `json:"name"` + URL string `json:"browser_download_url"` + Size int64 `json:"size"` +} + +func (d Driver) githubResolve(ctx context.Context, repository string) error { + release, err := d.getRelease(ctx, repository, "latest") + if err != nil { + return err + } + version := strings.TrimPrefix(strings.TrimSpace(release.TagName), "v") + if _, err := semver.StrictNewVersion(version); err != nil { + return errors.New("GitHub latest release is not a stable semantic version") + } + return d.output(version) +} + +func (d Driver) githubStage(ctx context.Context, repository, binary, version, _ string, stage, path string) error { + version = strings.TrimPrefix(version, "v") + if _, err := semver.StrictNewVersion(version); err != nil { + return errors.New("GitHub release version is invalid") + } + if err := resetStage(stage); err != nil { + return err + } + release, err := d.getRelease(ctx, repository, "tags/v"+version) + if err != nil { + release, err = d.getRelease(ctx, repository, "tags/"+version) + } + if err != nil { + return err + } + asset, checksums, err := selectReleaseAssets(release.Assets, d.goos(), d.goarch()) + if err != nil { + return err + } + archive, err := d.download(ctx, asset) + if err != nil { + return err + } + checksumData, err := d.download(ctx, checksums) + if err != nil { + return err + } + if err := verifyChecksum(asset.Name, archive, checksumData); err != nil { + return err + } + if err := extractTarBinary(archive, binary, filepath.Join(stage, "next")); err != nil { + return err + } + return backupPath(path, filepath.Join(stage, "previous")) +} + +func (d Driver) githubRollback(ctx context.Context, repository, binary, version, stage, path string) error { + previous := filepath.Join(stage, "previous") + if info, err := os.Stat(previous); err == nil && info.Mode().IsRegular() { + return replacePath(previous, path) + } + if _, err := semver.StrictNewVersion(strings.TrimPrefix(version, "v")); err != nil { + return errors.New("GitHub rollback version is unavailable") + } + temporary, err := os.MkdirTemp(filepath.Dir(path), ".tooltend-github-rollback-*") + if err != nil { + return err + } + defer os.RemoveAll(temporary) + if err := d.githubStage(ctx, repository, binary, version, "", temporary, path); err != nil { + return err + } + return replacePath(filepath.Join(temporary, "next"), path) +} + +func (d Driver) getRelease(ctx context.Context, repository, endpoint string) (githubRelease, error) { + if strings.Count(repository, "/") != 1 || strings.ContainsAny(repository, "\x00\r\n?#") { + return githubRelease{}, errors.New("GitHub repository identity is invalid") + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.github.com/repos/"+repository+"/releases/"+endpoint, nil) + if err != nil { + return githubRelease{}, err + } + request.Header.Set("Accept", "application/vnd.github+json") + request.Header.Set("User-Agent", "tooltend-bundle-driver") + if token := d.githubToken(ctx); token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := d.client().Do(request) + if err != nil { + return githubRelease{}, errors.New("GitHub release lookup failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return githubRelease{}, fmt.Errorf("GitHub release lookup failed with status %d", response.StatusCode) + } + var release githubRelease + decoder := json.NewDecoder(io.LimitReader(response.Body, 4<<20)) + if decoder.Decode(&release) != nil || release.TagName == "" { + return githubRelease{}, errors.New("GitHub release metadata is invalid") + } + return release, nil +} + +func (d Driver) githubToken(ctx context.Context) string { + for _, name := range []string{"GITHUB_TOKEN", "GH_TOKEN"} { + if token := strings.TrimSpace(os.Getenv(name)); validToken(token) { + return token + } + } + result, err := d.runner().Run(ctx, "gh", "auth", "token") + if err != nil { + return "" + } + token := strings.TrimSpace(string(result.Stdout)) + if !validToken(token) { + return "" + } + return token +} + +func validToken(value string) bool { + return value != "" && len(value) <= 4096 && !strings.ContainsAny(value, "\x00\r\n \t") +} + +func (d Driver) download(ctx context.Context, asset githubAsset) ([]byte, error) { + if asset.Size <= 0 || asset.Size > maxDownloadBytes || !strings.HasPrefix(asset.URL, "https://github.com/") { + return nil, errors.New("GitHub release asset metadata is invalid") + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, asset.URL, nil) + if err != nil { + return nil, err + } + request.Header.Set("User-Agent", "tooltend-bundle-driver") + response, err := d.client().Do(request) + if err != nil { + return nil, errors.New("GitHub release asset download failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GitHub release asset download failed with status %d", response.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(response.Body, maxDownloadBytes+1)) + if err != nil || int64(len(data)) > maxDownloadBytes { + return nil, errors.New("GitHub release asset exceeded the download limit") + } + if int64(len(data)) != asset.Size { + return nil, errors.New("GitHub release asset size mismatch") + } + return data, nil +} + +func (d Driver) client() *http.Client { + if d.Client != nil { + return d.Client + } + return &http.Client{Timeout: 5 * time.Minute} +} + +func (d Driver) goos() string { + if d.GOOS != "" { + return d.GOOS + } + return runtime.GOOS +} + +func (d Driver) goarch() string { + if d.GOARCH != "" { + return d.GOARCH + } + return runtime.GOARCH +} + +func selectReleaseAssets(assets []githubAsset, goos, goarch string) (githubAsset, githubAsset, error) { + osTokens := map[string][]string{"darwin": {"darwin", "apple-darwin"}, "linux": {"linux", "unknown-linux"}}[goos] + archTokens := map[string][]string{"arm64": {"arm64", "aarch64"}, "amd64": {"amd64", "x86_64"}}[goarch] + if len(osTokens) == 0 || len(archTokens) == 0 { + return githubAsset{}, githubAsset{}, errors.New("platform is not supported by the bundle release driver") + } + var candidates []githubAsset + var checksums githubAsset + for _, asset := range assets { + name := strings.ToLower(asset.Name) + if name == "checksums.txt" { + checksums = asset + continue + } + if strings.HasSuffix(name, ".tar.gz") && containsAny(name, osTokens) && containsAny(name, archTokens) { + candidates = append(candidates, asset) + } + } + if len(candidates) != 1 || checksums.Name == "" { + return githubAsset{}, githubAsset{}, errors.New("release does not contain one matching archive and checksums.txt") + } + return candidates[0], checksums, nil +} + +func containsAny(value string, candidates []string) bool { + for _, candidate := range candidates { + if strings.Contains(value, candidate) { + return true + } + } + return false +} + +func verifyChecksum(name string, data, checksums []byte) error { + expected := "" + for _, line := range strings.Split(string(checksums), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && strings.TrimPrefix(fields[len(fields)-1], "*") == name { + expected = strings.ToLower(fields[0]) + break + } + } + if len(expected) != sha256.Size*2 { + return errors.New("release checksum entry is missing") + } + digest := sha256.Sum256(data) + if hex.EncodeToString(digest[:]) != expected { + return errors.New("release checksum verification failed") + } + return nil +} + +func extractTarBinary(archive []byte, binary, target string) error { + reader, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + return errors.New("release archive is not valid gzip") + } + defer reader.Close() + tr := tar.NewReader(reader) + for { + header, nextErr := tr.Next() + if errors.Is(nextErr, io.EOF) { + break + } + if nextErr != nil { + return errors.New("release archive is invalid") + } + if header.Typeflag != tar.TypeReg || filepath.Base(filepath.Clean(header.Name)) != binary { + continue + } + if header.Size <= 0 || header.Size > 128<<20 { + return errors.New("release binary size is invalid") + } + data, readErr := io.ReadAll(io.LimitReader(tr, header.Size+1)) + if readErr != nil || int64(len(data)) != header.Size { + return errors.New("release binary is truncated") + } + return safeio.AtomicWriteFile(target, data, 0o755) + } + return errors.New("release archive does not contain the expected binary") +} + +func (d Driver) gitResolve(ctx context.Context, repository string) error { + result, err := d.runner().Run(ctx, "git", "ls-remote", repository, "HEAD") + if err != nil { + return errors.New("git source lookup failed") + } + fields := strings.Fields(string(result.Stdout)) + if len(fields) < 2 || !commitHash(fields[0]) { + return errors.New("git source did not resolve to an exact commit") + } + return d.output("git:" + strings.ToLower(fields[0])) +} + +func (d Driver) gitReleaseResolve(ctx context.Context, repository, githubRepository string) error { + release, err := d.getRelease(ctx, githubRepository, "latest") + if err != nil { + return err + } + tag := strings.TrimSpace(release.TagName) + if tag == "" || strings.ContainsAny(tag, "\x00\r\n") { + return errors.New("GitHub release tag is invalid") + } + for _, ref := range []string{"refs/tags/" + tag + "^{}", "refs/tags/" + tag} { + result, resolveErr := d.runner().Run(ctx, "git", "ls-remote", repository, ref) + if resolveErr != nil { + continue + } + fields := strings.Fields(string(result.Stdout)) + if len(fields) >= 2 && commitHash(fields[0]) { + return d.output("git:" + strings.ToLower(fields[0])) + } + } + return errors.New("GitHub release tag did not resolve to an exact git commit") +} + +func (d Driver) gitNPMReleaseResolve(ctx context.Context, repository, packageName string) error { + version, err := d.npmVersion(ctx, packageName) + if err != nil { + return err + } + for _, tag := range []string{"v" + version, version} { + for _, ref := range []string{"refs/tags/" + tag + "^{}", "refs/tags/" + tag} { + result, resolveErr := d.runner().Run(ctx, "git", "ls-remote", repository, ref) + if resolveErr != nil { + continue + } + fields := strings.Fields(string(result.Stdout)) + if len(fields) >= 2 && commitHash(fields[0]) { + return d.output("git:" + strings.ToLower(fields[0])) + } + } + } + return errors.New("npm version did not resolve to an exact git release tag") +} + +func (d Driver) gitStage(ctx context.Context, repository, subdir, ref, _ string, stage, path string) error { + commit := strings.TrimPrefix(ref, "git:") + if !commitHash(commit) { + return errors.New("git target ref is invalid") + } + cleanSubdir, err := sourceSubdir(subdir) + if err != nil { + return err + } + if err := resetStage(stage); err != nil { + return err + } + clone := filepath.Join(stage, "repository") + commands := [][]string{ + {"init", "--quiet", clone}, + {"-C", clone, "remote", "add", "origin", repository}, + {"-C", clone, "fetch", "--quiet", "--depth", "1", "origin", commit}, + {"-C", clone, "checkout", "--quiet", "--detach", "FETCH_HEAD"}, + } + for _, command := range commands { + if _, err := d.runner().Run(ctx, "git", command...); err != nil { + return errors.New("git skill staging failed") + } + } + root := clone + if cleanSubdir != "." { + root = filepath.Join(clone, filepath.FromSlash(cleanSubdir)) + } + if err := copySkillTree(root, filepath.Join(stage, "next")); err != nil { + return err + } + if err := skillHealth(filepath.Join(stage, "next")); err != nil { + return err + } + if err := backupPath(path, filepath.Join(stage, "previous")); err != nil { + return err + } + return backupSkillLock(path, stage) +} + +func (d Driver) gitRollback(ctx context.Context, repository, subdir, ref, stage, path string) error { + previous := filepath.Join(stage, "previous") + if info, err := os.Stat(previous); err == nil && info.IsDir() { + if err := replacePath(previous, path); err != nil { + return err + } + return restoreSkillLock(path, stage) + } + if !commitHash(strings.TrimPrefix(ref, "git:")) { + return errors.New("git rollback ref is unavailable") + } + temporary, err := os.MkdirTemp(filepath.Dir(path), ".tooltend-git-rollback-*") + if err != nil { + return err + } + defer os.RemoveAll(temporary) + if err := d.gitStage(ctx, repository, subdir, ref, "", temporary, path); err != nil { + return err + } + if err := replacePath(filepath.Join(temporary, "next"), path); err != nil { + return err + } + return detachSkillLock(path) +} + +func commitHash(value string) bool { + if len(value) != 40 { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +func sourceSubdir(value string) (string, error) { + value = filepath.ToSlash(filepath.Clean(strings.TrimSpace(value))) + if value == "" { + value = "." + } + if filepath.IsAbs(value) || value == ".." || strings.HasPrefix(value, "../") { + return "", errors.New("git skill subdirectory is invalid") + } + return value, nil +} + +func resetStage(stage string) error { + if !filepath.IsAbs(stage) || filepath.Clean(stage) == string(filepath.Separator) { + return errors.New("bundle stage path must be an absolute non-root path") + } + if err := os.RemoveAll(stage); err != nil { + return err + } + return os.MkdirAll(stage, 0o700) +} + +func backupPath(path, destination string) error { + if strings.TrimSpace(path) == "" { + return nil + } + if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + return copyPath(path, destination, false) +} + +func replacePath(source, destination string) error { + if !filepath.IsAbs(destination) || filepath.Clean(destination) == string(filepath.Separator) { + return errors.New("managed installation path must be an absolute non-root path") + } + if _, err := os.Lstat(source); err != nil { + return errors.New("staged installation is missing") + } + parent := filepath.Dir(destination) + if err := os.MkdirAll(parent, 0o755); err != nil { + return err + } + temporary, err := os.MkdirTemp(parent, ".tooltend-next-*") + if err != nil { + return err + } + _ = os.Remove(temporary) + defer os.RemoveAll(temporary) + if err := copyPath(source, temporary, false); err != nil { + return err + } + backup := temporary + ".old" + hadDestination := false + if _, err := os.Lstat(destination); err == nil { + hadDestination = true + if err := os.Rename(destination, backup); err != nil { + return err + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.Rename(temporary, destination); err != nil { + if hadDestination { + _ = os.Rename(backup, destination) + } + return err + } + return os.RemoveAll(backup) +} + +func copySkillTree(source, destination string) error { + if info, err := os.Stat(source); err != nil || !info.IsDir() { + return errors.New("git skill source directory is missing") + } + return copyPath(source, destination, true) +} + +func copyPath(source, destination string, skipGit bool) error { + info, err := os.Lstat(source) + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(source) + if err != nil { + return err + } + if filepath.IsAbs(target) || strings.HasPrefix(filepath.Clean(target), "..") { + return errors.New("source contains an unsafe symbolic link") + } + return os.Symlink(target, destination) + } + if info.IsDir() { + if err := os.MkdirAll(destination, info.Mode().Perm()); err != nil { + return err + } + entries, err := os.ReadDir(source) + if err != nil { + return err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + for _, entry := range entries { + if skipGit && entry.Name() == ".git" { + continue + } + if err := copyPath(filepath.Join(source, entry.Name()), filepath.Join(destination, entry.Name()), skipGit); err != nil { + return err + } + } + return nil + } + if !info.Mode().IsRegular() { + return errors.New("source contains an unsupported filesystem entry") + } + data, err := os.ReadFile(source) + if err != nil { + return err + } + return safeio.AtomicWriteFile(destination, data, info.Mode().Perm()) +} + +func skillHealth(path string) error { + info, err := os.Stat(filepath.Join(path, "SKILL.md")) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 || info.Size() > 4<<20 { + return errors.New("managed skill is missing a valid SKILL.md") + } + return nil +} + +func skillLockPath(path string) string { + parent := filepath.Dir(path) + if filepath.Base(parent) != "skills" || filepath.Base(filepath.Dir(parent)) != ".agents" { + return "" + } + return filepath.Join(filepath.Dir(parent), ".skill-lock.json") +} + +func backupSkillLock(path, stage string) error { + lock := skillLockPath(path) + if lock == "" { + return nil + } + manifest := map[string]bool{"existed": false} + if info, err := os.Stat(lock); err == nil && info.Mode().IsRegular() { + manifest["existed"] = true + if err := copyPath(lock, filepath.Join(stage, "skill-lock.previous"), false); err != nil { + return err + } + } + data, _ := json.Marshal(manifest) + return safeio.AtomicWriteFile(filepath.Join(stage, "skill-lock.json"), data, 0o600) +} + +func detachSkillLock(path string) error { + lock := skillLockPath(path) + if lock == "" { + return nil + } + data, err := os.ReadFile(lock) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var document struct { + Version int `json:"version"` + Skills map[string]json.RawMessage `json:"skills"` + Dismissed map[string]json.RawMessage `json:"dismissed"` + } + if json.Unmarshal(data, &document) != nil || document.Skills == nil { + return errors.New("npx skills lock file is invalid") + } + delete(document.Skills, filepath.Base(path)) + updated, err := json.MarshalIndent(document, "", " ") + if err != nil { + return err + } + updated = append(updated, '\n') + return safeio.AtomicWriteFile(lock, updated, 0o600) +} + +func restoreSkillLock(path, stage string) error { + manifestData, err := os.ReadFile(filepath.Join(stage, "skill-lock.json")) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var manifest map[string]bool + if json.Unmarshal(manifestData, &manifest) != nil { + return errors.New("skill lock rollback metadata is invalid") + } + lock := skillLockPath(path) + if lock == "" { + return nil + } + if !manifest["existed"] { + if err := os.Remove(lock); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + return copyPath(filepath.Join(stage, "skill-lock.previous"), lock, false) +} + +var mainlineHookFiles = []string{".claude/settings.json", ".codex/config.toml", ".codex/hooks.json", ".cursor/hooks.json"} + +func stageMainlineHooks(project, stage string) error { + if !filepath.IsAbs(project) { + return errors.New("mainline hook project path must be absolute") + } + if err := resetStage(stage); err != nil { + return err + } + existed := map[string]bool{} + for _, relative := range mainlineHookFiles { + source := filepath.Join(project, filepath.FromSlash(relative)) + if info, err := os.Stat(source); err == nil && info.Mode().IsRegular() { + existed[relative] = true + if err := copyPath(source, filepath.Join(stage, "previous", filepath.FromSlash(relative)), false); err != nil { + return err + } + } + } + data, _ := json.Marshal(existed) + return safeio.AtomicWriteFile(filepath.Join(stage, "manifest.json"), data, 0o600) +} + +func restoreMainlineHooks(project, stage string) (bool, error) { + data, err := os.ReadFile(filepath.Join(stage, "manifest.json")) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + var existed map[string]bool + if json.Unmarshal(data, &existed) != nil { + return false, errors.New("mainline hook rollback metadata is invalid") + } + for _, relative := range mainlineHookFiles { + target := filepath.Join(project, filepath.FromSlash(relative)) + if existed[relative] { + if err := copyPath(filepath.Join(stage, "previous", filepath.FromSlash(relative)), target, false); err != nil { + return false, err + } + } else if err := os.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { + return false, err + } + } + return true, nil +} + +func (d Driver) runMainlineHooks(ctx context.Context, project, action string) error { + if !filepath.IsAbs(project) { + return errors.New("mainline hook project path must be absolute") + } + runner := d.runner() + if value, ok := runner.(execx.ExecRunner); ok { + value.Dir = project + runner = value + } + if _, err := runner.Run(ctx, "mainline", "hooks", action); err != nil { + return errors.New("mainline hook command failed") + } + return nil +} diff --git a/internal/bundledriver/driver_test.go b/internal/bundledriver/driver_test.go new file mode 100644 index 0000000..30ce9c5 --- /dev/null +++ b/internal/bundledriver/driver_test.go @@ -0,0 +1,145 @@ +package bundledriver + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/z2z23n0/tooltend/internal/safeio" +) + +func TestSelectReleaseAssetsSupportsGoAndRustNames(t *testing.T) { + tests := []struct { + name string + asset string + goos string + goarch string + }{ + {name: "go", asset: "mainline_0.5.0_darwin_arm64.tar.gz", goos: "darwin", goarch: "arm64"}, + {name: "rust", asset: "xsearch-x86_64-unknown-linux-gnu.tar.gz", goos: "linux", goarch: "amd64"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + selected, checksums, err := selectReleaseAssets([]githubAsset{ + {Name: test.asset, URL: "https://github.com/example/repo/releases/download/v1/" + test.asset, Size: 10}, + {Name: "checksums.txt", URL: "https://github.com/example/repo/releases/download/v1/checksums.txt", Size: 10}, + }, test.goos, test.goarch) + if err != nil { + t.Fatal(err) + } + if selected.Name != test.asset || checksums.Name != "checksums.txt" { + t.Fatalf("selected = %#v, checksums = %#v", selected, checksums) + } + }) + } +} + +func TestVerifyChecksumRejectsTampering(t *testing.T) { + data := []byte("release") + digest := sha256.Sum256(data) + checksums := []byte(hex.EncodeToString(digest[:]) + " tool.tar.gz\n") + if err := verifyChecksum("tool.tar.gz", data, checksums); err != nil { + t.Fatal(err) + } + if err := verifyChecksum("tool.tar.gz", []byte("tampered"), checksums); err == nil { + t.Fatal("expected tampered asset rejection") + } +} + +func TestGitSkillActivationDetachesAndCompensationRestoresSkillLock(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".agents", "skills", "mainline") + stage := filepath.Join(home, "stage") + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + if err := safeio.AtomicWriteFile(filepath.Join(path, "SKILL.md"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + lockPath := filepath.Join(home, ".agents", ".skill-lock.json") + lock := map[string]any{"version": 3, "skills": map[string]any{"mainline": map[string]any{"source": "mainline-org/mainline"}}, "dismissed": map[string]any{}} + lockData, _ := json.Marshal(lock) + if err := safeio.AtomicWriteFile(lockPath, lockData, 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(stage, "next"), 0o755); err != nil { + t.Fatal(err) + } + if err := safeio.AtomicWriteFile(filepath.Join(stage, "next", "SKILL.md"), []byte("new"), 0o644); err != nil { + t.Fatal(err) + } + if err := backupPath(path, filepath.Join(stage, "previous")); err != nil { + t.Fatal(err) + } + if err := backupSkillLock(path, stage); err != nil { + t.Fatal(err) + } + driver := Driver{} + if err := driver.Execute(context.Background(), []string{"git-activate", stage, path}); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(path, "SKILL.md"), "new") + updatedLock, err := os.ReadFile(lockPath) + if err != nil { + t.Fatal(err) + } + if string(updatedLock) == string(lockData) { + t.Fatal("npx skills lock entry was not detached") + } + if err := driver.Execute(context.Background(), []string{"git-rollback", "unused", ".", "", stage, path}); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(path, "SKILL.md"), "old") + restoredLock, err := os.ReadFile(lockPath) + if err != nil { + t.Fatal(err) + } + var restored struct { + Skills map[string]json.RawMessage `json:"skills"` + } + if json.Unmarshal(restoredLock, &restored) != nil || restored.Skills["mainline"] == nil { + t.Fatal("npx skills lock was not restored during compensation") + } +} + +func TestMainlineHookBackupRestoresRemovedAndExistingFiles(t *testing.T) { + project := t.TempDir() + stage := filepath.Join(t.TempDir(), "stage") + existing := filepath.Join(project, ".codex", "hooks.json") + if err := safeio.AtomicWriteFile(existing, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + if err := stageMainlineHooks(project, stage); err != nil { + t.Fatal(err) + } + if err := safeio.AtomicWriteFile(existing, []byte("new"), 0o644); err != nil { + t.Fatal(err) + } + created := filepath.Join(project, ".cursor", "hooks.json") + if err := safeio.AtomicWriteFile(created, []byte("created"), 0o644); err != nil { + t.Fatal(err) + } + restored, err := restoreMainlineHooks(project, stage) + if err != nil || !restored { + t.Fatalf("restored = %t, err = %v", restored, err) + } + assertFileContent(t, existing, "old") + if _, err := os.Stat(created); !os.IsNotExist(err) { + t.Fatalf("new hook file still exists: %v", err) + } +} + +func assertFileContent(t *testing.T, path, expected string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(data) != expected { + t.Fatalf("%s = %q, want %q", path, data, expected) + } +} diff --git a/internal/cli/app.go b/internal/cli/app.go index 988b7fe..2855c6a 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -92,6 +92,9 @@ func New(options Options) *cobra.Command { flags.BoolVar(&a.global.NoColor, "no-color", false, "disable colored human output") root.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error { a.warnings = nil + if cmd.Annotations[internalDriverAnnotation] == "true" { + return nil + } if legacyCommand(commandName(cmd)) { a.warnings = append(a.warnings, v1.Warning{Code: "deprecated_component_api", Message: "this component-level command is deprecated; use tooltend bundles instead"}) } @@ -139,6 +142,7 @@ func New(options Options) *cobra.Command { a.newReconcileCommand(), a.newWatchdogCommand(), a.newVersionCommand(), + a.newBundleDriverCommand(), ) root.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error { return a.writeFailure(commandName(cmd), cliError("invalid_argument", err.Error(), err)) diff --git a/internal/cli/bundle_commands.go b/internal/cli/bundle_commands.go index 2f40971..2788289 100644 --- a/internal/cli/bundle_commands.go +++ b/internal/cli/bundle_commands.go @@ -285,7 +285,9 @@ func (a *App) newBundlesUpdateCommand() *cobra.Command { if prepareErr != nil { return nil, prepareErr } - previews = append(previews, preview) + if preview.UpdateAvailable { + previews = append(previews, preview) + } } var results []bundle.UpdateResult value := plan.Plan{ID: "bundle-update-v1", Title: "Update complete ToolTend bundles"} diff --git a/internal/cli/bundle_driver_commands.go b/internal/cli/bundle_driver_commands.go new file mode 100644 index 0000000..ada351a --- /dev/null +++ b/internal/cli/bundle_driver_commands.go @@ -0,0 +1,24 @@ +package cli + +import ( + "github.com/spf13/cobra" + + "github.com/z2z23n0/tooltend/internal/bundledriver" +) + +const internalDriverAnnotation = "tooltend.io/internal-bundle-driver" + +func (a *App) newBundleDriverCommand() *cobra.Command { + command := &cobra.Command{ + Use: "__bundle-driver [arguments...]", + Hidden: true, + DisableFlagParsing: true, + Args: cobra.MinimumNArgs(1), + Annotations: map[string]string{internalDriverAnnotation: "true"}, + } + command.RunE = func(cmd *cobra.Command, args []string) error { + driver := bundledriver.Driver{Runner: a.runner, Out: a.out} + return driver.Execute(cmd.Context(), args) + } + return command +} diff --git a/internal/cli/worker_commands.go b/internal/cli/worker_commands.go index 95aeb4e..18e9784 100644 --- a/internal/cli/worker_commands.go +++ b/internal/cli/worker_commands.go @@ -350,6 +350,9 @@ func (a *App) reconcileOnce(ctx context.Context, paths config.Paths, reason stri if prepareErr != nil { return prepareErr } + if !preview.UpdateAvailable { + return nil + } if !activate { return database.UpsertBundleRelease(bundleCtx, preview.Target) } diff --git a/internal/store/bundles.go b/internal/store/bundles.go index 04b490d..41e3976 100644 --- a/internal/store/bundles.go +++ b/internal/store/bundles.go @@ -428,6 +428,10 @@ func (s *Store) ConfigureBundle(ctx context.Context, value model.BundlePolicy) e _, err = tx.ExecContext(ctx, `INSERT INTO bundle_policies(bundle_id,mode,recipe_trusted,updated_at) VALUES(?,?,?,?) ON CONFLICT(bundle_id) DO UPDATE SET mode=excluded.mode,recipe_trusted=excluded.recipe_trusted,updated_at=excluded.updated_at`, value.BundleID, value.Mode, boolInt(value.RecipeTrusted), timeText(value.UpdatedAt)) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `UPDATE installations SET managed=? WHERE bundle_id=?`, boolInt(value.Mode == model.BundlePolicyAuto || value.Mode == model.BundlePolicyManual), value.BundleID) return err }) } diff --git a/internal/store/bundles_test.go b/internal/store/bundles_test.go index 9f2af2b..7ea371c 100644 --- a/internal/store/bundles_test.go +++ b/internal/store/bundles_test.go @@ -7,6 +7,9 @@ import ( "sort" "strconv" "testing" + "time" + + "github.com/z2z23n0/tooltend/internal/model" ) func TestSchemaV6MigratesV4WithBackup(t *testing.T) { @@ -60,3 +63,35 @@ func TestSchemaV6MigratesV4WithBackup(t *testing.T) { t.Fatalf("migration backups = %v err=%v", backups, err) } } + +func TestConfigureBundleMarksPhysicalInstallationsManaged(t *testing.T) { + database, err := OpenRW(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + ctx := context.Background() + now := time.Now().UTC() + bundle := model.Bundle{ID: "bundle", Slug: "bundle", Name: "Bundle", RecipeID: "bundle", RecipeVersion: "1", RecipeSource: "builtin", Owner: model.LifecycleDelegated, ConfigState: model.BundleUnconfigured, Confidence: model.BundleConfidenceHigh, DiscoveredAt: now, LastSeenAt: now} + if err := database.UpsertBundle(ctx, bundle); err != nil { + t.Fatal(err) + } + installation := model.Installation{ID: "installation", BundleID: bundle.ID, Driver: "git-skill", Path: "/tmp/skill", Owner: model.LifecycleDelegated, LastSeenAt: now} + if err := database.UpsertInstallation(ctx, installation); err != nil { + t.Fatal(err) + } + if err := database.ConfigureBundle(ctx, model.BundlePolicy{BundleID: bundle.ID, Mode: model.BundlePolicyAuto, RecipeTrusted: true, UpdatedAt: now}); err != nil { + t.Fatal(err) + } + installations, err := database.ListInstallations(ctx, bundle.ID) + if err != nil || len(installations) != 1 || !installations[0].Managed { + t.Fatalf("installations = %#v, err = %v", installations, err) + } + if err := database.ConfigureBundle(ctx, model.BundlePolicy{BundleID: bundle.ID, Mode: model.BundlePolicyObserve, RecipeTrusted: true, UpdatedAt: now.Add(time.Second)}); err != nil { + t.Fatal(err) + } + installations, err = database.ListInstallations(ctx, bundle.ID) + if err != nil || installations[0].Managed { + t.Fatalf("observed installations = %#v, err = %v", installations, err) + } +} From 2f2ce52aa3ea4a5607b84d604a034d53f3814420 Mon Sep 17 00:00:00 2001 From: z2z23n0 Date: Wed, 15 Jul 2026 23:03:17 +0800 Subject: [PATCH 4/5] fix: include Pi hook in Mainline rollback --- internal/bundledriver/driver.go | 8 +++++++- internal/bundledriver/driver_test.go | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/internal/bundledriver/driver.go b/internal/bundledriver/driver.go index 54a63a1..f3b5aed 100644 --- a/internal/bundledriver/driver.go +++ b/internal/bundledriver/driver.go @@ -849,7 +849,13 @@ func restoreSkillLock(path, stage string) error { return copyPath(filepath.Join(stage, "skill-lock.previous"), lock, false) } -var mainlineHookFiles = []string{".claude/settings.json", ".codex/config.toml", ".codex/hooks.json", ".cursor/hooks.json"} +var mainlineHookFiles = []string{ + ".claude/settings.json", + ".codex/config.toml", + ".codex/hooks.json", + ".cursor/hooks.json", + ".pi/extensions/mainline.ts", +} func stageMainlineHooks(project, stage string) error { if !filepath.IsAbs(project) { diff --git a/internal/bundledriver/driver_test.go b/internal/bundledriver/driver_test.go index 30ce9c5..713249f 100644 --- a/internal/bundledriver/driver_test.go +++ b/internal/bundledriver/driver_test.go @@ -123,6 +123,10 @@ func TestMainlineHookBackupRestoresRemovedAndExistingFiles(t *testing.T) { if err := safeio.AtomicWriteFile(created, []byte("created"), 0o644); err != nil { t.Fatal(err) } + createdPi := filepath.Join(project, ".pi", "extensions", "mainline.ts") + if err := safeio.AtomicWriteFile(createdPi, []byte("created"), 0o644); err != nil { + t.Fatal(err) + } restored, err := restoreMainlineHooks(project, stage) if err != nil || !restored { t.Fatalf("restored = %t, err = %v", restored, err) @@ -131,6 +135,9 @@ func TestMainlineHookBackupRestoresRemovedAndExistingFiles(t *testing.T) { if _, err := os.Stat(created); !os.IsNotExist(err) { t.Fatalf("new hook file still exists: %v", err) } + if _, err := os.Stat(createdPi); !os.IsNotExist(err) { + t.Fatalf("new Pi hook file still exists: %v", err) + } } func assertFileContent(t *testing.T, path, expected string) { From 78445b3064c1820ebd2813e2df0eb06a39ef982f Mon Sep 17 00:00:00 2001 From: z2z23n0 Date: Sat, 18 Jul 2026 12:29:59 +0800 Subject: [PATCH 5/5] fix: restore bundle updates and native notifications --- README.md | 1 + install.sh | 6 + internal/cli/app.go | 1 + internal/cli/cli_test.go | 3 +- internal/cli/notifier_commands.go | 41 +++++ internal/cli/self_repair.go | 7 + internal/cli/worker_commands.go | 20 ++- internal/doctor/doctor.go | 15 ++ internal/notify/desktop.go | 39 ++++- internal/notify/desktop_test.go | 21 ++- internal/notify/install.go | 164 +++++++++++++++++++ internal/notify/install_test.go | 43 +++++ internal/notify/macos/ToolTendNotifier.swift | 103 ++++++++++++ scripts/install.sh | 6 + 14 files changed, 451 insertions(+), 19 deletions(-) create mode 100644 internal/cli/notifier_commands.go create mode 100644 internal/notify/install.go create mode 100644 internal/notify/install_test.go create mode 100644 internal/notify/macos/ToolTendNotifier.swift diff --git a/README.md b/README.md index 04abef8..694d57e 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ SessionStart / ToolUse / 每日任务 / 用户命令 - `kick` 只启动一个脱离当前会话的一次性 worker。全局文件锁保证并发 Session 不会并行更新。 - macOS 使用 launchd，Linux 使用 systemd user timer；两者每天启动一次 `reconcile --once`，没有常驻 ToolTend 进程。 - 每轮 reconcile 都会持久化完整运行状态；主任务之后由独立 watchdog 检查漏跑、失败或未完成状态。失败默认发送桌面通知，并在下次 Codex/Claude SessionStart 时补充提醒。 +- macOS 安装器会用 Xcode Command Line Tools 构建并把 `ToolTend Notifier.app` 注册到 `~/Applications`，首次发送时需要在系统提示中允许通知；不再借用 Script Editor 的通知身份，`tooltend doctor` 也会检查安装与授权状态。 - macOS 调度输出保存在 `~/.local/state/tooltend/logs/`，不会再丢弃到 `/dev/null`；`tooltend status` 和 `tooltend doctor` 会显示最近一次完整 reconcile 的结果。 - 未执行 `bundles configure` 的 Bundle 不检查更新、不下载，也不调用安装器。 - Bundle 更新先完成所有 Artifact 的解析、校验和 staging，再按物理 Installation 激活；失败时按相反顺序补偿。 diff --git a/install.sh b/install.sh index c7086b2..6cc9769 100755 --- a/install.sh +++ b/install.sh @@ -63,6 +63,12 @@ if [[ -f "$target" && ! -L "$target" ]]; then fi mv -f "$binary" "$target" +if [[ "$os_name" == "darwin" ]]; then + if ! "$target" __notifier install; then + echo "ToolTend was installed, but macOS notifications need Xcode Command Line Tools. Install them and run: tooltend __notifier install" >&2 + fi +fi + echo "Installed tooltend to $target" case ":$PATH:" in *":$INSTALL_DIR:"*) ;; diff --git a/internal/cli/app.go b/internal/cli/app.go index 2855c6a..16891e6 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -143,6 +143,7 @@ func New(options Options) *cobra.Command { a.newWatchdogCommand(), a.newVersionCommand(), a.newBundleDriverCommand(), + a.newNotifierCommand(), ) root.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error { return a.writeFailure(commandName(cmd), cliError("invalid_argument", err.Error(), err)) diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 837677e..9397496 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -22,6 +22,7 @@ import ( "github.com/z2z23n0/tooltend/internal/inventory" "github.com/z2z23n0/tooltend/internal/lockfile" "github.com/z2z23n0/tooltend/internal/model" + "github.com/z2z23n0/tooltend/internal/notify" "github.com/z2z23n0/tooltend/internal/reconcile" "github.com/z2z23n0/tooltend/internal/store" ) @@ -357,7 +358,7 @@ func TestScheduledFailureSendsDesktopNotification(t *testing.T) { a.notifyScheduledOutcome(context.Background(), paths, reconcile.RunResult{Failed: 2, FailureNotificationQueued: true}, nil) wantName := "notify-send" if runtime.GOOS == "darwin" { - wantName = "/usr/bin/osascript" + wantName = notify.DarwinNotifierExecutable(home) } if runner.name != wantName || !strings.Contains(strings.Join(runner.args, " "), "2 task(s) failed") { t.Fatalf("notification call = %s %#v", runner.name, runner.args) diff --git a/internal/cli/notifier_commands.go b/internal/cli/notifier_commands.go new file mode 100644 index 0000000..dcd7204 --- /dev/null +++ b/internal/cli/notifier_commands.go @@ -0,0 +1,41 @@ +package cli + +import ( + "fmt" + "runtime" + + "github.com/spf13/cobra" + + "github.com/z2z23n0/tooltend/internal/notify" +) + +func (a *App) newNotifierCommand() *cobra.Command { + command := &cobra.Command{ + Use: "__notifier [arguments...]", + Hidden: true, + Annotations: map[string]string{internalDriverAnnotation: "true"}, + Args: cobra.MinimumNArgs(1), + } + command.RunE = func(cmd *cobra.Command, args []string) error { + switch args[0] { + case "install": + if len(args) != 1 { + return fmt.Errorf("notifier install accepts no arguments") + } + result, err := notify.InstallDarwin(cmd.Context(), a.home, a.runner) + if err != nil { + return err + } + _, err = fmt.Fprintf(a.out, "Installed ToolTend Notifier to %s\n", result.AppPath) + return err + case "send": + if len(args) != 3 { + return fmt.Errorf("notifier send requires title and message") + } + return a.desktopNotifier().Send(cmd.Context(), args[1], args[2]) + default: + return fmt.Errorf("unsupported notifier action %q on %s", args[0], runtime.GOOS) + } + } + return command +} diff --git a/internal/cli/self_repair.go b/internal/cli/self_repair.go index 700fb99..bec84bc 100644 --- a/internal/cli/self_repair.go +++ b/internal/cli/self_repair.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "path/filepath" + "runtime" v1 "github.com/z2z23n0/tooltend/internal/api/v1" "github.com/z2z23n0/tooltend/internal/config" "github.com/z2z23n0/tooltend/internal/host" + "github.com/z2z23n0/tooltend/internal/notify" "github.com/z2z23n0/tooltend/internal/scheduler" ) @@ -46,5 +48,10 @@ func (a *App) repairAfterSelfUpdate(ctx context.Context, paths config.Paths) []v if err != nil { warnings = append(warnings, v1.Warning{Code: "self_update_scheduler_repair_failed", Message: fmt.Sprintf("self-update applied, but the ToolTend scheduler needs repair: %s", err)}) } + if runtime.GOOS == "darwin" { + if _, err := notify.InstallDarwin(ctx, a.home, a.runner); err != nil { + warnings = append(warnings, v1.Warning{Code: "self_update_notifier_repair_failed", Message: fmt.Sprintf("self-update applied, but ToolTend Notifier needs repair: %s", err)}) + } + } return warnings } diff --git a/internal/cli/worker_commands.go b/internal/cli/worker_commands.go index 18e9784..2741630 100644 --- a/internal/cli/worker_commands.go +++ b/internal/cli/worker_commands.go @@ -198,15 +198,15 @@ func (a *App) newWatchdogCommand() *cobra.Command { if a.global.DryRun { return map[string]any{"dry_run": true, "max_age": maxAge.String(), "state_dir": paths.StateDir}, nil } - desktop := notify.Desktop{Runner: a.runner} + desktop := a.desktopNotifier() cfg, err := config.Load(paths.ConfigFile) if err != nil { - _ = desktop.Send(ctx, "ToolTend", "Scheduled update state cannot be checked. Run `tooltend doctor` for details.") + a.sendDesktopNotification(ctx, "Scheduled update state cannot be checked. Run `tooltend doctor` for details.") return nil, err } database, err := store.OpenRW(paths.DatabaseFile) if err != nil { - _ = desktop.Send(ctx, "ToolTend", "Scheduled update state cannot be opened. Run `tooltend doctor` for details.") + a.sendDesktopNotification(ctx, "Scheduled update state cannot be opened. Run `tooltend doctor` for details.") return nil, err } defer database.Close() @@ -235,10 +235,22 @@ func (a *App) notifyScheduledOutcome(ctx context.Context, paths config.Paths, va message = fmt.Sprintf("Scheduled update completed: %d task(s) succeeded.", result.Succeeded) } if message != "" { - _ = (notify.Desktop{Runner: a.runner}).Send(ctx, "ToolTend", message) + a.sendDesktopNotification(ctx, message) } } +func (a *App) desktopNotifier() notify.Desktop { + return notify.Desktop{AppPath: notify.DarwinNotifierExecutable(a.home), Runner: a.runner} +} + +func (a *App) sendDesktopNotification(ctx context.Context, message string) bool { + if err := a.desktopNotifier().Send(ctx, "ToolTend", message); err != nil { + _, _ = fmt.Fprintf(a.errOut, "Warning: desktop notification failed: %v\n", err) + return false + } + return true +} + func (a *App) reconcileOnce(ctx context.Context, paths config.Paths, reason string) (any, error) { cfg, err := config.Load(paths.ConfigFile) if err != nil { diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 28abfe7..74a3412 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -22,6 +22,7 @@ import ( "github.com/z2z23n0/tooltend/internal/lifecycle" "github.com/z2z23n0/tooltend/internal/lockfile" "github.com/z2z23n0/tooltend/internal/model" + "github.com/z2z23n0/tooltend/internal/notify" "github.com/z2z23n0/tooltend/internal/plan" "github.com/z2z23n0/tooltend/internal/scheduler" "github.com/z2z23n0/tooltend/internal/store" @@ -166,6 +167,20 @@ func RunWithOptions(ctx context.Context, options Options) Report { if options.Home == "" || options.Executable == "" { return report } + if runtime.GOOS == "darwin" && options.Config.Notify.Mode != model.NotifyNone { + check := Check{Name: "desktop_notifier"} + if err := notify.CheckDarwin(options.Home); err != nil { + check.Level = LevelWarning + check.Message = "macOS desktop notifier is unavailable; rerun the ToolTend installer" + } else if err := notify.CheckDarwinAuthorization(ctx, options.Home, options.Runner); err != nil { + check.Level = LevelWarning + check.Message = "macOS desktop notification permission is not allowed; enable ToolTend Notifier in System Settings" + } else { + check.Level = LevelOK + check.Message = "macOS desktop notifier is installed and authorized" + } + report.Checks = append(report.Checks, check) + } agents := options.Agents if len(agents) == 0 { agents = []model.HostKind{model.HostCodex, model.HostClaude} diff --git a/internal/notify/desktop.go b/internal/notify/desktop.go index 5f7c332..2b8f4a0 100644 --- a/internal/notify/desktop.go +++ b/internal/notify/desktop.go @@ -3,6 +3,8 @@ package notify import ( "context" "errors" + "fmt" + "path/filepath" "runtime" "strings" @@ -10,10 +12,18 @@ import ( ) var ErrUnsupported = errors.New("desktop notifications are not supported on this platform") +var ErrNotifierUnavailable = errors.New("ToolTend Notifier is not installed") + +const ( + DarwinAppName = "ToolTend Notifier.app" + DarwinBundleID = "io.tooltend.notifier.native" + DarwinExecutable = "applet" +) type Desktop struct { - GOOS string - Runner execx.Runner + GOOS string + AppPath string + Runner execx.Runner } func (d Desktop) Send(ctx context.Context, title, message string) error { @@ -30,9 +40,19 @@ func (d Desktop) Send(ctx context.Context, title, message string) error { } switch goos { case "darwin": - script := "display notification " + appleScriptString(message) + " with title " + appleScriptString(title) - _, err := runner.Run(ctx, "/usr/bin/osascript", "-e", script) - return err + path := strings.TrimSpace(d.AppPath) + if path == "" || !filepath.IsAbs(path) { + return ErrNotifierUnavailable + } + result, err := runner.Run(ctx, path, title, message) + if err == nil { + return nil + } + detail := strings.TrimSpace(string(result.Stderr)) + if detail == "" { + return fmt.Errorf("desktop notification: %w", err) + } + return fmt.Errorf("desktop notification: %s: %w", detail, err) case "linux": _, err := runner.Run(ctx, "notify-send", "--app-name=ToolTend", title, message) return err @@ -41,7 +61,10 @@ func (d Desktop) Send(ctx context.Context, title, message string) error { } } -func appleScriptString(value string) string { - value = strings.NewReplacer("\\", "\\\\", "\"", "\\\"", "\r", " ", "\n", " ").Replace(value) - return "\"" + value + "\"" +func DarwinAppPath(home string) string { + return filepath.Join(home, "Applications", DarwinAppName) +} + +func DarwinNotifierExecutable(home string) string { + return filepath.Join(DarwinAppPath(home), "Contents", "MacOS", DarwinExecutable) } diff --git a/internal/notify/desktop_test.go b/internal/notify/desktop_test.go index c449c07..5c41f7d 100644 --- a/internal/notify/desktop_test.go +++ b/internal/notify/desktop_test.go @@ -2,7 +2,8 @@ package notify import ( "context" - "strings" + "errors" + "path/filepath" "testing" "github.com/z2z23n0/tooltend/internal/execx" @@ -18,16 +19,24 @@ func (r *recordingRunner) Run(_ context.Context, name string, args ...string) (e return execx.Result{}, nil } -func TestDarwinNotificationEscapesAppleScript(t *testing.T) { +func TestDarwinNotificationUsesStableAppIdentity(t *testing.T) { runner := &recordingRunner{} - err := (Desktop{GOOS: "darwin", Runner: runner}).Send(context.Background(), `Tool"Tend`, "line 1\nline 2") + appPath := filepath.Join(t.TempDir(), "ToolTend Notifier") + err := (Desktop{GOOS: "darwin", AppPath: appPath, Runner: runner}).Send(context.Background(), `Tool"Tend`, "line 1\nline 2") if err != nil { t.Fatal(err) } - if runner.name != "/usr/bin/osascript" || len(runner.args) != 2 || runner.args[0] != "-e" { + if runner.name != appPath || len(runner.args) != 2 { t.Fatalf("call = %s %#v", runner.name, runner.args) } - if strings.Contains(runner.args[1], "\n") || !strings.Contains(runner.args[1], `Tool\"Tend`) { - t.Fatalf("unsafe script = %q", runner.args[1]) + if runner.args[0] != `Tool"Tend` || runner.args[1] != "line 1\nline 2" { + t.Fatalf("notification arguments = %#v", runner.args) + } +} + +func TestDarwinNotificationRequiresInstalledNotifier(t *testing.T) { + err := (Desktop{GOOS: "darwin", Runner: &recordingRunner{}}).Send(context.Background(), "ToolTend", "message") + if !errors.Is(err, ErrNotifierUnavailable) { + t.Fatalf("error = %v", err) } } diff --git a/internal/notify/install.go b/internal/notify/install.go new file mode 100644 index 0000000..e23f66c --- /dev/null +++ b/internal/notify/install.go @@ -0,0 +1,164 @@ +package notify + +import ( + "context" + _ "embed" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/z2z23n0/tooltend/internal/execx" +) + +const launchServicesRegister = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" + +//go:embed macos/ToolTendNotifier.swift +var darwinNotifierSource []byte + +const darwinInfoPlist = ` + + + + CFBundleExecutable + applet + CFBundleIdentifier + io.tooltend.notifier.native + CFBundleName + ToolTend Notifier + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSUIElement + + + +` + +type InstallResult struct { + AppPath string `json:"app_path"` + Executable string `json:"executable"` +} + +func CheckDarwin(home string) error { + if runtime.GOOS != "darwin" { + return ErrUnsupported + } + appPath := DarwinAppPath(home) + executable := DarwinNotifierExecutable(home) + info, err := os.Stat(executable) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return ErrNotifierUnavailable + } + plist, err := os.ReadFile(filepath.Join(appPath, "Contents", "Info.plist")) + if err != nil || !strings.Contains(string(plist), DarwinBundleID) { + return errors.New("ToolTend Notifier has an invalid application identity") + } + return nil +} + +func CheckDarwinAuthorization(ctx context.Context, home string, runner execx.Runner) error { + if err := CheckDarwin(home); err != nil { + return err + } + if runner == nil { + runner = execx.ExecRunner{} + } + result, err := runner.Run(ctx, DarwinNotifierExecutable(home), "--check") + if err != nil { + return commandFailure("check notifier authorization", result, err) + } + return nil +} + +func InstallDarwin(ctx context.Context, home string, runner execx.Runner) (InstallResult, error) { + result := InstallResult{AppPath: DarwinAppPath(home), Executable: DarwinNotifierExecutable(home)} + if runtime.GOOS != "darwin" { + return result, ErrUnsupported + } + if strings.TrimSpace(home) == "" || !filepath.IsAbs(home) { + return result, errors.New("notifier install requires an absolute home directory") + } + if runner == nil { + runner = execx.ExecRunner{} + } + applicationsDir := filepath.Join(home, "Applications") + if err := os.MkdirAll(applicationsDir, 0o755); err != nil { + return result, fmt.Errorf("create Applications directory: %w", err) + } + buildDir, err := os.MkdirTemp(applicationsDir, ".tooltend-notifier-build-") + if err != nil { + return result, fmt.Errorf("create notifier build directory: %w", err) + } + defer os.RemoveAll(buildDir) + buildApp := filepath.Join(buildDir, DarwinAppName) + contentsDir := filepath.Join(buildApp, "Contents") + macOSDir := filepath.Join(contentsDir, "MacOS") + if err := os.MkdirAll(macOSDir, 0o755); err != nil { + return result, fmt.Errorf("create notifier bundle: %w", err) + } + sourcePath := filepath.Join(buildDir, "ToolTendNotifier.swift") + if err := os.WriteFile(sourcePath, darwinNotifierSource, 0o600); err != nil { + return result, fmt.Errorf("write notifier source: %w", err) + } + plistPath := filepath.Join(contentsDir, "Info.plist") + if err := os.WriteFile(plistPath, []byte(darwinInfoPlist), 0o644); err != nil { + return result, fmt.Errorf("write notifier application identity: %w", err) + } + if commandResult, commandErr := runner.Run(ctx, "/usr/bin/xcrun", "--sdk", "macosx", "swiftc", "-framework", "AppKit", "-framework", "UserNotifications", "-o", resultExecutable(buildApp), sourcePath); commandErr != nil { + return result, commandFailure("compile notifier", commandResult, commandErr) + } + if commandResult, commandErr := runner.Run(ctx, "/usr/bin/codesign", "--force", "--sign", "-", buildApp); commandErr != nil { + return result, commandFailure("sign notifier", commandResult, commandErr) + } + backupPath := filepath.Join(applicationsDir, fmt.Sprintf(".tooltend-notifier-backup-%d", time.Now().UnixNano())) + hadPrevious := false + if _, statErr := os.Lstat(result.AppPath); statErr == nil { + if err := os.Rename(result.AppPath, backupPath); err != nil { + return result, fmt.Errorf("back up existing notifier: %w", err) + } + hadPrevious = true + } else if !errors.Is(statErr, os.ErrNotExist) { + return result, fmt.Errorf("inspect existing notifier: %w", statErr) + } + restore := func() { + _ = os.RemoveAll(result.AppPath) + if hadPrevious { + _ = os.Rename(backupPath, result.AppPath) + } + } + if err := os.Rename(buildApp, result.AppPath); err != nil { + restore() + return result, fmt.Errorf("install notifier: %w", err) + } + if commandResult, commandErr := runner.Run(ctx, launchServicesRegister, "-f", result.AppPath); commandErr != nil { + restore() + return result, commandFailure("register notifier", commandResult, commandErr) + } + if err := CheckDarwin(home); err != nil { + restore() + return result, err + } + if hadPrevious { + _ = os.RemoveAll(backupPath) + } + return result, nil +} + +func resultExecutable(appPath string) string { + return filepath.Join(appPath, "Contents", "MacOS", DarwinExecutable) +} + +func commandFailure(action string, result execx.Result, err error) error { + detail := strings.TrimSpace(string(result.Stderr)) + if detail == "" { + return fmt.Errorf("%s: %w", action, err) + } + return fmt.Errorf("%s: %s: %w", action, detail, err) +} diff --git a/internal/notify/install_test.go b/internal/notify/install_test.go new file mode 100644 index 0000000..7e26891 --- /dev/null +++ b/internal/notify/install_test.go @@ -0,0 +1,43 @@ +package notify + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestDarwinBundleMetadataMatchesConstants(t *testing.T) { + if !strings.Contains(darwinInfoPlist, ""+DarwinBundleID+"") { + t.Fatalf("Info.plist does not contain bundle id %q", DarwinBundleID) + } + if len(darwinNotifierSource) == 0 || !strings.Contains(string(darwinNotifierSource), "UNUserNotificationCenter") { + t.Fatal("native notifier source is missing") + } +} + +func TestCheckDarwinAuthorizationRunsInstalledHelper(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS-only notifier") + } + home := t.TempDir() + executable := DarwinNotifierExecutable(home) + if err := os.MkdirAll(filepath.Dir(executable), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(executable, []byte("helper"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(DarwinAppPath(home), "Contents", "Info.plist"), []byte(""+DarwinBundleID+""), 0o644); err != nil { + t.Fatal(err) + } + runner := &recordingRunner{} + if err := CheckDarwinAuthorization(context.Background(), home, runner); err != nil { + t.Fatal(err) + } + if runner.name != executable || len(runner.args) != 1 || runner.args[0] != "--check" { + t.Fatalf("authorization check = %s %#v", runner.name, runner.args) + } +} diff --git a/internal/notify/macos/ToolTendNotifier.swift b/internal/notify/macos/ToolTendNotifier.swift new file mode 100644 index 0000000..833469d --- /dev/null +++ b/internal/notify/macos/ToolTendNotifier.swift @@ -0,0 +1,103 @@ +import AppKit +import Foundation +import UserNotifications + +final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCenterDelegate { + private let arguments = Array(CommandLine.arguments.dropFirst()) + private var finished = false + + func applicationDidFinishLaunching(_ notification: Notification) { + DispatchQueue.main.asyncAfter(deadline: .now() + 30) { + self.fail("notification request timed out", code: 75) + } + let center = UNUserNotificationCenter.current() + if arguments == ["--check"] { + checkAuthorization(with: center) + return + } + guard arguments.count == 2 else { + fail("expected title and message", code: 64) + return + } + + center.delegate = self + center.getNotificationSettings { settings in + switch settings.authorizationStatus { + case .notDetermined: + center.requestAuthorization(options: [.alert]) { granted, error in + if let error { + self.fail("notification authorization failed: \(error)", code: 77) + } else if !granted { + self.fail("notification permission denied", code: 77) + } else { + self.deliver(with: center) + } + } + case .authorized, .provisional, .ephemeral: + self.deliver(with: center) + case .denied: + self.fail("notification permission denied; allow ToolTend Notifier in System Settings", code: 77) + @unknown default: + self.fail("unknown notification authorization state", code: 70) + } + } + } + + private func checkAuthorization(with center: UNUserNotificationCenter) { + center.getNotificationSettings { settings in + switch settings.authorizationStatus { + case .authorized, .provisional, .ephemeral: + self.finish(code: 0) + case .notDetermined: + self.fail("notification permission has not been requested", code: 77) + case .denied: + self.fail("notification permission denied; allow ToolTend Notifier in System Settings", code: 77) + @unknown default: + self.fail("unknown notification authorization state", code: 70) + } + } + } + + func userNotificationCenter( + _ center: UNUserNotificationCenter, + willPresent notification: UNNotification, + withCompletionHandler completionHandler: @escaping (UNNotificationPresentationOptions) -> Void + ) { + completionHandler([.banner, .list]) + } + + private func deliver(with center: UNUserNotificationCenter) { + let content = UNMutableNotificationContent() + content.title = arguments[0] + content.body = arguments[1] + let request = UNNotificationRequest(identifier: UUID().uuidString, content: content, trigger: nil) + center.add(request) { error in + if let error { + self.fail("notification delivery failed: \(error)", code: 70) + return + } + self.finish(code: 0, delay: 2) + } + } + + private func fail(_ message: String, code: Int32) { + FileHandle.standardError.write(Data((message + "\n").utf8)) + finish(code: code) + } + + private func finish(code: Int32, delay: TimeInterval = 0) { + DispatchQueue.main.async { + guard !self.finished else { return } + self.finished = true + DispatchQueue.main.asyncAfter(deadline: .now() + delay) { + exit(code) + } + } + } +} + +let app = NSApplication.shared +let delegate = AppDelegate() +app.delegate = delegate +app.setActivationPolicy(.accessory) +app.run() diff --git a/scripts/install.sh b/scripts/install.sh index 1cf79a6..3800e08 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -22,6 +22,12 @@ chmod 0755 "$tmp" mv -f "$tmp" "$INSTALL_DIR/tooltend" trap - EXIT +if [[ "$(uname -s)" == "Darwin" ]]; then + if ! "$INSTALL_DIR/tooltend" __notifier install; then + echo "ToolTend was installed, but macOS notifications need Xcode Command Line Tools. Install them and run: tooltend __notifier install" >&2 + fi +fi + echo "Installed tooltend to $INSTALL_DIR/tooltend" case ":$PATH:" in *":$INSTALL_DIR:"*) ;;