diff --git a/CHANGELOG.md b/CHANGELOG.md index b976a19..4987ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A new ProxyRack device stayed off the account.** ProxyRack only knows a new UUID about five minutes after its client first connects, and the deploy made its one attempt to add it before that, so the device earned for nobody. A "try again later" answer is now stored and retried, 6 minutes after the deploy and then twice as long each time up to 4 hours; a restart keeps it. After a day the app stops and tells you to add the device by hand. The deploy output says it is still being tried. - **A copy of ProxyRack reported as the same device as the original.** Only device names built from the hostname got a suffix, so a proxied copy or a second copy on the box reused the original's device UUID and two IPs fought over one ProxyRack device. Ids the catalog marks `per_device` now get a value of their own for each copy, derived the same way as in the web app (same length and case, stable across redeploys); a copy whose id cannot be derived, such as EarnApp's, is refused instead of cloning it. - **UpRock said you could cash out at $5.** Its minimum is 250 UPT. The catalog now says so, and UPT gets a price from CoinGecko, so the app can compare an UpRock balance with that minimum. USDT and USDC get prices too, so a cash-out recorded in either counts toward the totals. - **A copy the cap reset could not start stayed stopped for a whole period.** It kept its "stopped by the cap" mark, and nothing tried again until the next reset or a Start by hand. The once-a-minute pass now starts such a copy while the proxy is uncapped, and leaves alone copies you stopped yourself. diff --git a/app.go b/app.go index 02e13d7..4d9dd8f 100644 --- a/app.go +++ b/app.go @@ -6,11 +6,13 @@ import ( "encoding/base64" "errors" "fmt" + "log" "net" neturl "net/url" "os" "path/filepath" stdruntime "runtime" + "slices" "sort" "strconv" "strings" @@ -101,6 +103,8 @@ type App struct { usageStop func() // proxyStop stops the proxied-copies pass (app_proxies.go) and waits for it. proxyStop func() + // deviceLinkStop stops the device-registration retry pass (app_device_links.go). + deviceLinkStop func() // activity is the Activity sampler (activity_sampler.go): off until the owner switches it on. activity activityState } @@ -127,7 +131,7 @@ type collectorRegistry interface { // collectorRegistry need not implement it. type deviceRegistry interface { Unlinked(slug string) []string - RegisterDevice(ctx context.Context, slug string, credentials map[string]string) bool + RegisterDevice(ctx context.Context, slug string, credentials map[string]string) collectors.Registration } func NewApp() *App { @@ -254,6 +258,8 @@ func (a *App) startCore(ctx context.Context) error { a.startUsageSampler(ctx) // Proxied copies: traffic per proxy, the cap, and earners whose tunnel came back. a.startProxyTicker(ctx) + // Device registrations the provider asked to retry later. + a.startDeviceLinkRetry(ctx) // What each earner talks to, when the owner switched Activity on. a.startActivitySampler(ctx) // Optional, and a no-op unless the user paired this Desktop with a @@ -296,6 +302,7 @@ func (a *App) Shutdown(_ context.Context) { a.stopScheduler() a.stopUsageSampler() a.stopProxyTicker() + a.stopDeviceLinkRetry() a.stopActivitySampler() a.stopUpstream() if a.fleetAPI != nil { @@ -356,6 +363,9 @@ type AppState struct { // AddedDevices is, per slug, the device id the last deploy added to the user's // account itself (ProxyRack with its API key saved), so no manual step is needed. AddedDevices map[string]string `json:"addedDevices"` + // RetryingDevices is, per slug, the device id the provider did not know yet at + // deploy time and the app keeps trying to add to the account for a day. + RetryingDevices map[string]string `json:"retryingDevices"` // StateBackups lists the deployed services whose catalog entry marks state as // irreplaceable, and whether the user has a backup of it. StateBackups []statebackup.Row `json:"stateBackups"` @@ -550,6 +560,7 @@ func (a *App) GetAppState() (AppState, error) { Scaling: a.services.ScalingLimits(visible), Copies: a.services.Copies(), AddedDevices: a.addedDeviceIDs(), + RetryingDevices: a.retryingDeviceIDs(), StateBackups: a.stateBackups(deployments), Drift: a.containerDrift(), Isolation: a.isolationStatus(), @@ -670,26 +681,44 @@ func (a *App) addedDeviceIDs() map[string]string { // registerDevice adds a just-deployed device to the user's account where the // provider's API can (ProxyRack, with its API key among the credentials). It never // fails the deploy, which has already happened: anything short of "on the account" -// leaves the manual step in the deploy output. +// leaves the manual step in the deploy output, except a device the provider does +// not know yet, which is stored and retried (app_device_links.go). func (a *App) registerDevice(slug string, creds map[string]string) { devicesReg, ok := a.collectors.(deviceRegistry) svc, inCatalog := a.catalog.Get(slug) if !ok || !inCatalog || svc.DeviceLink == nil { return } + id := strings.TrimSpace(creds[svc.DeviceLink.IDEnv]) + a.dropOtherDeviceLinks(slug, id) ctx, cancel := context.WithTimeout(a.ctx, 20*time.Second) defer cancel() - added := devicesReg.RegisterDevice(ctx, slug, creds) - a.addedMu.Lock() - defer a.addedMu.Unlock() - if !added { + result := devicesReg.RegisterDevice(ctx, slug, creds) + if result == collectors.RegisterPending && a.store != nil { + now := time.Now() + if err := a.store.SavePendingDeviceLink(slug, id, now, now.Add(deviceLinkFirstRetry)); err != nil { + log.Printf("device links: store %s: %v", slug, err) + } + } + if result == collectors.RegisterAdded && a.store != nil { + _ = a.store.DeletePendingDeviceLink(slug, id) + } + if result != collectors.RegisterAdded { + a.addedMu.Lock() delete(a.addedDevices, slug) + a.addedMu.Unlock() return } + a.setAddedDevice(slug, id) +} + +func (a *App) setAddedDevice(slug, id string) { + a.addedMu.Lock() + defer a.addedMu.Unlock() if a.addedDevices == nil { a.addedDevices = map[string]string{} } - a.addedDevices[slug] = strings.TrimSpace(creds[svc.DeviceLink.IDEnv]) + a.addedDevices[slug] = id } // GetEarningsSummary returns the converted, aggregated earnings summary on its @@ -1865,6 +1894,7 @@ func (a *App) notifications(status runtime.Status, earnings []store.EarningsReco } } items = append(items, a.unlinkedDeviceNotifications(deployments)...) + items = append(items, a.deviceLinkGaveUpNotifications()...) items = append(items, a.proxyNotifications()...) items = append(items, a.activityNotifications()...) if len(deployments) == 0 { @@ -1881,9 +1911,12 @@ func (a *App) unlinkedDeviceNotifications(deployments []store.Deployment) []Noti if !ok || a.catalog == nil { return nil } + // A device with a stored registration is being retried, or has its own + // give-up notice (app_device_links.go). + pending := a.pendingDeviceLinkIDs() var items []Notification for _, dep := range deployments { - ids := devicesReg.Unlinked(dep.Slug) + ids := slices.DeleteFunc(devicesReg.Unlinked(dep.Slug), func(id string) bool { return pending[dep.Slug][id] }) svc, inCatalog := a.catalog.Get(dep.Slug) if len(ids) == 0 || !inCatalog || svc.DeviceLink == nil { continue diff --git a/app_device_links.go b/app_device_links.go new file mode 100644 index 0000000..c078b44 --- /dev/null +++ b/app_device_links.go @@ -0,0 +1,232 @@ +package main + +import ( + "context" + "log" + neturl "net/url" + "slices" + "strings" + "time" + + "github.com/GeiserX/CashPilot-Desktop/internal/collectors" +) + +// A registration the provider asked to retry is tried again this long after the +// first refusal, then twice as long each time up to the cap; after the give-up the +// user is told to add the device by hand. The same numbers as the web app. +const ( + deviceLinkFirstRetry = 6 * time.Minute + deviceLinkMaxRetry = 4 * time.Hour + deviceLinkGiveUp = 24 * time.Hour + // deviceLinkTick is how often the retry pass looks for a registration that is due. + deviceLinkTick = 3 * time.Minute +) + +// deployedDevice says whether slug is deployed here with a device link, and the +// device id its saved credentials hold. An error means the store could not be read, +// which proves nothing either way. +func (a *App) deployedDevice(slug string) (id string, deployed bool, err error) { + if a.store == nil || a.catalog == nil { + return "", false, nil + } + svc, ok := a.catalog.Get(slug) + if !ok || svc.DeviceLink == nil { + return "", false, nil + } + if _, deployed, err = a.store.GetDeployment(slug); err != nil || !deployed { + return "", false, err + } + creds, err := a.store.GetCredentials(slug) + if err != nil { + return "", false, err + } + return strings.TrimSpace(creds[svc.DeviceLink.IDEnv]), true, nil +} + +// runsDevice is whether slug is deployed here with deviceID; false when unsure. +func (a *App) runsDevice(slug, deviceID string) bool { + id, deployed, err := a.deployedDevice(slug) + return err == nil && deployed && id == deviceID +} + +// dropOtherDeviceLinks forgets slug's pending registrations of any device but id: +// a deploy replaced that device. +func (a *App) dropOtherDeviceLinks(slug, id string) { + if a.store == nil { + return + } + rows, err := a.store.ListPendingDeviceLinks() + if err != nil { + return + } + for _, row := range rows { + if row.Slug == slug && row.DeviceID != id { + _ = a.store.DeletePendingDeviceLink(row.Slug, row.DeviceID) + } + } +} + +// pendingDeviceLinkIDs is, per slug, every device id with a stored registration, +// retrying or given up. +func (a *App) pendingDeviceLinkIDs() map[string]map[string]bool { + out := map[string]map[string]bool{} + if a.store == nil { + return out + } + rows, err := a.store.ListPendingDeviceLinks() + if err != nil { + return out + } + for _, row := range rows { + if out[row.Slug] == nil { + out[row.Slug] = map[string]bool{} + } + out[row.Slug][row.DeviceID] = true + } + return out +} + +// retryDeviceLinks retries the registrations a provider asked to try again later, +// with backoff, and gives up after a day. A service no longer deployed here is +// forgotten without asking the provider. A row whose id the saved credentials no +// longer hold is left alone: credentials saved without a redeploy leave the old +// device running, and a deploy drops the row itself (dropOtherDeviceLinks). +func (a *App) retryDeviceLinks(ctx context.Context, now time.Time) { + devicesReg, ok := a.collectors.(deviceRegistry) + if !ok || a.store == nil { + return + } + rows, err := a.store.ListPendingDeviceLinks() + if err != nil { + log.Printf("device links: list: %v", err) + return + } + for _, row := range rows { + if ctx.Err() != nil { + return + } + id, deployed, err := a.deployedDevice(row.Slug) + if err != nil { + continue + } + if !deployed { + _ = a.store.DeletePendingDeviceLink(row.Slug, row.DeviceID) + continue + } + if id != row.DeviceID || row.GaveUp || row.NextAt.After(now) { + continue + } + creds, err := a.store.GetCredentials(row.Slug) + if err != nil { + continue + } + callCtx, cancel := context.WithTimeout(ctx, 20*time.Second) + result := devicesReg.RegisterDevice(callCtx, row.Slug, creds) + cancel() + if result == collectors.RegisterAdded { + _ = a.store.DeletePendingDeviceLink(row.Slug, row.DeviceID) + a.store.RecordEvent(row.Slug, "device_linked", row.DeviceID+" added to the account") + // A redeploy during the call may have put another device in its place. + if a.runsDevice(row.Slug, row.DeviceID) { + a.setAddedDevice(row.Slug, row.DeviceID) + } + continue + } + attempts := row.Attempts + 1 + if now.Sub(row.FirstAt) >= deviceLinkGiveUp { + _ = a.store.UpdatePendingDeviceLink(row.Slug, row.DeviceID, attempts, row.NextAt, true) + continue + } + wait := deviceLinkMaxRetry + if attempts < 16 { // past that the doubling is far over the cap anyway + wait = min(deviceLinkFirstRetry< 0 { - time.Sleep(f.hold) + if f.gate != nil { + <-f.gate } f.inFlight.Add(-1) @@ -584,20 +586,42 @@ func TestCollectAllCollectsCredentialOnlyServicesAndDedups(t *testing.T) { // (Within that one run the services now collect CONCURRENTLY, bounded by // collectConcurrency, so max-concurrent-collects may exceed 1 — that is expected and is // a separate property, asserted only to be within the bound.) +// +// The run that wins is parked on the fake's gate until every other call has +// returned. A sleep used to stand in for that, and on a slow runner a call could +// start after the first run had finished, which is a legitimate second run and +// failed the test with 6 collects. func TestCollectAllSingleFlight(t *testing.T) { fake := newFakeCollector([]string{"svc-a", "svc-b", "svc-c"}, nil) - fake.hold = 2 * time.Millisecond // hold each collect so overlapping runs would be observable + fake.gate = make(chan struct{}) app, _, cancel := newSchedulerTestApp(t, fake, "svc-a", "svc-b", "svc-c") defer cancel() + const callers = 8 + returned := make(chan struct{}, callers) var wg sync.WaitGroup - for i := 0; i < 8; i++ { + for i := 0; i < callers; i++ { wg.Add(1) go func() { defer wg.Done() app.collectAll(context.Background()) + returned <- struct{}{} }() } + + // A call can only return while the gate is shut by bailing on the guard, so + // callers-1 returns means exactly one run holds the guard and is parked. + deadline := time.After(10 * time.Second) + for got := 0; got < callers-1; got++ { + select { + case <-returned: + case <-deadline: + close(fake.gate) + wg.Wait() + t.Fatalf("single-flight violated: only %d of %d calls bailed while one run was in progress", got, callers-1) + } + } + close(fake.gate) wg.Wait() // Single-flight: exactly one collectAll body ran, so the 3 services were collected diff --git a/device_link_app_test.go b/device_link_app_test.go index 05b8c06..5ea6357 100644 --- a/device_link_app_test.go +++ b/device_link_app_test.go @@ -7,6 +7,7 @@ import ( "testing/fstest" "github.com/GeiserX/CashPilot-Desktop/internal/catalog" + "github.com/GeiserX/CashPilot-Desktop/internal/collectors" "github.com/GeiserX/CashPilot-Desktop/internal/runtime" "github.com/GeiserX/CashPilot-Desktop/internal/store" ) @@ -15,13 +16,13 @@ import ( type fakeDevices struct { *fakeCollector unlinked map[string][]string - register bool + register collectors.Registration asked []string } func (f *fakeDevices) Unlinked(slug string) []string { return f.unlinked[slug] } -func (f *fakeDevices) RegisterDevice(_ context.Context, slug string, _ map[string]string) bool { +func (f *fakeDevices) RegisterDevice(_ context.Context, slug string, _ map[string]string) collectors.Registration { f.asked = append(f.asked, slug) return f.register } @@ -68,7 +69,7 @@ func TestADeviceTheProviderDoesNotKnowIsAWarning(t *testing.T) { } func TestDeployRecordsADeviceTheProviderAdded(t *testing.T) { - devices := &fakeDevices{fakeCollector: newFakeCollector(nil, nil), register: true} + devices := &fakeDevices{fakeCollector: newFakeCollector(nil, nil), register: collectors.RegisterAdded} app := &App{ctx: context.Background(), catalog: deviceLinkCatalog(t), collectors: devices} app.registerDevice("linked", map[string]string{"UUID": " dev-1 "}) @@ -76,7 +77,7 @@ func TestDeployRecordsADeviceTheProviderAdded(t *testing.T) { t.Fatalf("addedDevices[linked] = %q, want dev-1", got) } - devices.register = false + devices.register = collectors.RegisterFailed app.registerDevice("linked", map[string]string{"UUID": "dev-1"}) if _, ok := app.addedDeviceIDs()["linked"]; ok { t.Fatal("a failed add must leave the manual step, not a stale 'added'") diff --git a/device_link_retry_test.go b/device_link_retry_test.go new file mode 100644 index 0000000..4d990c5 --- /dev/null +++ b/device_link_retry_test.go @@ -0,0 +1,256 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/GeiserX/CashPilot-Desktop/internal/collectors" + "github.com/GeiserX/CashPilot-Desktop/internal/store" +) + +// A device the provider does not know yet is registered later, not dropped. +// ProxyRack only knows a new UUID about five minutes after its client first +// connects, and /device/add answers "try again in 5 minutes" until then. The +// deploy used to make one attempt at that moment and never try again, so every +// new device stayed off the account and earned for nobody. + +const retryUUID = "dev-1" + +func retryApp(t *testing.T, register collectors.Registration) (*App, *fakeDevices) { + t.Helper() + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + if err := st.UpsertDeployment(store.Deployment{Slug: "linked", Name: "linked", Status: "running"}); err != nil { + t.Fatalf("UpsertDeployment: %v", err) + } + if err := st.SaveCredentials("linked", map[string]string{"UUID": retryUUID}); err != nil { + t.Fatalf("SaveCredentials: %v", err) + } + devices := &fakeDevices{fakeCollector: newFakeCollector(nil, nil), register: register} + return &App{ctx: context.Background(), catalog: deviceLinkCatalog(t), collectors: devices, store: st}, devices +} + +func pending(t *testing.T, app *App) []store.PendingDeviceLink { + t.Helper() + rows, err := app.store.ListPendingDeviceLinks() + if err != nil { + t.Fatalf("ListPendingDeviceLinks: %v", err) + } + return rows +} + +// due stores a registration whose next try has come, first refused at firstAt. +func due(t *testing.T, app *App, firstAt time.Time, attempts int) { + t.Helper() + if err := app.store.SavePendingDeviceLink("linked", retryUUID, firstAt, firstAt); err != nil { + t.Fatalf("SavePendingDeviceLink: %v", err) + } + if attempts > 0 { + if err := app.store.UpdatePendingDeviceLink("linked", retryUUID, attempts, firstAt, false); err != nil { + t.Fatalf("UpdatePendingDeviceLink: %v", err) + } + } +} + +func TestADeviceTheProviderDoesNotKnowYetIsStoredToRetry(t *testing.T) { + app, _ := retryApp(t, collectors.RegisterPending) + before := time.Now() + app.registerDevice("linked", map[string]string{"UUID": " " + retryUUID + " "}) + + rows := pending(t, app) + if len(rows) != 1 || rows[0].Slug != "linked" || rows[0].DeviceID != retryUUID || rows[0].GaveUp { + t.Fatalf("want one live pending row for the device, got %+v", rows) + } + // The first retry waits out the provider's five minutes. + if wait := rows[0].NextAt.Sub(before); wait < 5*time.Minute || wait > deviceLinkFirstRetry+time.Second { + t.Fatalf("first retry in %v, want just over five minutes", wait) + } + if got := app.retryingDeviceIDs()["linked"]; got != retryUUID { + t.Fatalf("retryingDevices[linked] = %q, want %s", got, retryUUID) + } + if _, ok := app.addedDeviceIDs()["linked"]; ok { + t.Fatal("a device the provider does not know yet is not added") + } + // No "add it by hand" warning while the app is still adding it. + app.collectors.(*fakeDevices).unlinked = map[string][]string{"linked": {retryUUID}} + if n := app.unlinkedDeviceNotifications([]store.Deployment{{Slug: "linked"}}); len(n) != 0 { + t.Fatalf("a device being retried got the manual-step warning: %+v", n) + } +} + +func TestARetryThatSucceedsMarksTheDeviceAdded(t *testing.T) { + app, devices := retryApp(t, collectors.RegisterAdded) + now := time.Now() + due(t, app, now.Add(-10*time.Minute), 0) + + app.retryDeviceLinks(context.Background(), now) + + if len(devices.asked) != 1 { + t.Fatalf("want one add, asked %v", devices.asked) + } + if rows := pending(t, app); len(rows) != 0 { + t.Fatalf("an added device is no longer pending, got %+v", rows) + } + if got := app.addedDeviceIDs()["linked"]; got != retryUUID { + t.Fatalf("addedDevices[linked] = %q, want %s", got, retryUUID) + } +} + +func TestARetryTheProviderStillRefusesBacksOff(t *testing.T) { + app, _ := retryApp(t, collectors.RegisterPending) + now := time.Now() + due(t, app, now.Add(-10*time.Minute), 0) + + app.retryDeviceLinks(context.Background(), now) + + rows := pending(t, app) + if len(rows) != 1 || rows[0].Attempts != 1 || rows[0].GaveUp { + t.Fatalf("want one live row after one attempt, got %+v", rows) + } + if wait := rows[0].NextAt.Sub(now.Truncate(time.Second)); wait != 2*deviceLinkFirstRetry { + t.Fatalf("next try in %v, want %v", wait, 2*deviceLinkFirstRetry) + } + + // The doubling stops at the cap. + if err := app.store.UpdatePendingDeviceLink("linked", retryUUID, 9, now, false); err != nil { + t.Fatal(err) + } + app.retryDeviceLinks(context.Background(), now) + if wait := pending(t, app)[0].NextAt.Sub(now.Truncate(time.Second)); wait != deviceLinkMaxRetry { + t.Fatalf("next try in %v, want the %v cap", wait, deviceLinkMaxRetry) + } +} + +func TestAfterADayItGivesUpAndTellsTheUser(t *testing.T) { + app, devices := retryApp(t, collectors.RegisterPending) + devices.unlinked = map[string][]string{"linked": {retryUUID}} + now := time.Now() + due(t, app, now.Add(-25*time.Hour), 8) + + app.retryDeviceLinks(context.Background(), now) + app.retryDeviceLinks(context.Background(), now.Add(time.Hour)) + + if rows := pending(t, app); len(rows) != 1 || !rows[0].GaveUp { + t.Fatalf("want the row marked given up, got %+v", rows) + } + if len(devices.asked) != 1 { + t.Fatalf("a given-up device is not tried again, asked %v", devices.asked) + } + if _, ok := app.retryingDeviceIDs()["linked"]; ok { + t.Fatal("a given-up device is no longer shown as retrying") + } + var found *Notification + for _, n := range app.deviceLinkGaveUpNotifications() { + found = &n + } + if found == nil || found.Level != "warning" || !strings.Contains(found.Message, retryUUID) || + !strings.Contains(found.Message, "Open the link.") || !strings.Contains(found.Message, "https://example.test/r/"+retryUUID) { + t.Fatalf("want a warning naming the device, its step and link, got %+v", found) + } + if n := app.unlinkedDeviceNotifications([]store.Deployment{{Slug: "linked"}}); len(n) != 0 { + t.Fatalf("one notice per device, but the generic warning fired too: %+v", n) + } + // The user adds it by hand: the provider's next check says it is on the account. + devices.unlinked = nil + if n := app.deviceLinkGaveUpNotifications(); len(n) != 0 { + t.Fatalf("a device now on the account still warned: %+v", n) + } +} + +func TestAServiceThatIsRemovedIsDroppedWithoutAsking(t *testing.T) { + app, devices := retryApp(t, collectors.RegisterAdded) + now := time.Now() + due(t, app, now.Add(-10*time.Minute), 0) + if err := app.store.DeleteDeployment("linked"); err != nil { + t.Fatal(err) + } + + app.retryDeviceLinks(context.Background(), now) + + if len(devices.asked) != 0 { + t.Fatalf("the provider must not be asked about a device that is gone, asked %v", devices.asked) + } + if rows := pending(t, app); len(rows) != 0 { + t.Fatalf("want the row dropped, got %+v", rows) + } +} + +func TestCredentialsSavedWithoutARedeployKeepTheRunningDevicesRetry(t *testing.T) { + app, devices := retryApp(t, collectors.RegisterAdded) + now := time.Now() + due(t, app, now.Add(-10*time.Minute), 0) + // The old container still runs dev-1; only the saved form changed. + if err := app.store.SaveCredentials("linked", map[string]string{"UUID": "dev-2"}); err != nil { + t.Fatal(err) + } + + app.retryDeviceLinks(context.Background(), now) + + if len(devices.asked) != 0 { + t.Fatalf("the saved id is not the running one, so nothing can be added; asked %v", devices.asked) + } + if rows := pending(t, app); len(rows) != 1 || rows[0].DeviceID != retryUUID { + t.Fatalf("the running device's retry must stay, got %+v", rows) + } + if _, ok := app.retryingDeviceIDs()["linked"]; ok { + t.Fatal("the form's id is not being retried, so the deploy output must not say it is") + } +} + +func TestARedeployWithAnotherDeviceDropsTheOldRetry(t *testing.T) { + app, _ := retryApp(t, collectors.RegisterFailed) + due(t, app, time.Now().Add(-10*time.Minute), 0) + + app.registerDevice("linked", map[string]string{"UUID": "dev-2"}) + + if rows := pending(t, app); len(rows) != 0 { + t.Fatalf("the replaced device's retry must go, got %+v", rows) + } +} + +func TestARedeployAfterGivingUpGetsANewDayOfRetries(t *testing.T) { + app, devices := retryApp(t, collectors.RegisterPending) + old := time.Now().Add(-25 * time.Hour) + due(t, app, old, 8) + app.retryDeviceLinks(context.Background(), time.Now()) // gives up + + app.registerDevice("linked", map[string]string{"UUID": retryUUID}) // the redeploy + rows := pending(t, app) + app.retryDeviceLinks(context.Background(), time.Now()) // not due yet + + if len(rows) != 1 || rows[0].GaveUp || rows[0].Attempts != 0 || !rows[0].FirstAt.After(old) || !rows[0].NextAt.After(time.Now()) { + t.Fatalf("want a fresh window, got %+v", rows) + } + if len(devices.asked) != 2 { + t.Fatalf("the give-up try and the redeploy, nothing more; asked %v", devices.asked) + } +} + +func TestADeployThatAddsTheDeviceClearsAPendingRetry(t *testing.T) { + app, _ := retryApp(t, collectors.RegisterPending) + app.registerDevice("linked", map[string]string{"UUID": retryUUID}) + app.collectors.(*fakeDevices).register = collectors.RegisterAdded + app.registerDevice("linked", map[string]string{"UUID": retryUUID}) + if rows := pending(t, app); len(rows) != 0 { + t.Fatalf("an added device is no longer pending, got %+v", rows) + } +} + +func TestTheRetryRunsOnItsOwnLoop(t *testing.T) { + app, devices := retryApp(t, collectors.RegisterAdded) + due(t, app, time.Now().Add(-10*time.Minute), 0) + app.startDeviceLinkRetry(context.Background()) + deadline := time.Now().Add(5 * time.Second) + for len(pending(t, app)) != 0 && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + app.stopDeviceLinkRetry() + if len(devices.asked) != 1 || len(pending(t, app)) != 0 { + t.Fatalf("the loop's first pass should add the due device, asked %v", devices.asked) + } +} diff --git a/frontend/scripts/details_check.mjs b/frontend/scripts/details_check.mjs index 18ada31..37b5e06 100644 --- a/frontend/scripts/details_check.mjs +++ b/frontend/scripts/details_check.mjs @@ -298,6 +298,16 @@ check( "an added id that is not this one keeps the manual step", deviceLinkText(earnapp, {EARNAPP_UUID: uuid}, "other").startsWith("Link this device"), ); +const retrying = deviceLinkText(earnapp, {EARNAPP_UUID: uuid}, "", uuid); +check( + "a device the app is still trying to add says so instead of the manual step, and keeps the link", + retrying.includes("keeps trying to add it") && !retrying.includes("Open this link") && retrying.includes(`https://earnapp.com/r/${uuid}`), + retrying, +); +check( + "a retrying id that is not this one keeps the manual step", + deviceLinkText(earnapp, {EARNAPP_UUID: uuid}, "", "other").includes("Open this link while signed in."), +); check("a service with no device link gets nothing", deviceLinkText(honeygain, {EARNAPP_UUID: uuid}) === ""); // --------------------------------------------------------------------------- diff --git a/frontend/src/main.ts b/frontend/src/main.ts index 4cc30ff..6240164 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -1048,7 +1048,7 @@ async function runImportDeploy(slug: string) { state = await GetAppState(); // The saved values hold the device id; the step is the one the wizard shows. const creds = await GetCredentials(slug).catch(() => ({} as Record)); - const link = deviceLinkText(state?.services?.find((service) => service.slug === slug), creds, state?.addedDevices?.[slug] || ""); + const link = deviceLinkText(state?.services?.find((service) => service.slug === slug), creds, state?.addedDevices?.[slug] || "", state?.retryingDevices?.[slug] || ""); if (link) importLinks = { ...importLinks, [slug]: link }; renderImportIfShown(); } catch (error) { @@ -1988,7 +1988,7 @@ async function runWizardAction(slug: string, action: string) { instancesInput.value = String(deployment.instances || 1); instancesInput.dataset.initial = String(deployment.instances || 1); } - const link = deviceLinkText(state?.services?.find((service) => service.slug === slug), values, state?.addedDevices?.[slug] || ""); + const link = deviceLinkText(state?.services?.find((service) => service.slug === slug), values, state?.addedDevices?.[slug] || "", state?.retryingDevices?.[slug] || ""); if (output) output.textContent = link ? `Deployed.\n\n${link}` : "Deployed."; return; } diff --git a/frontend/src/render/details.ts b/frontend/src/render/details.ts index c5b45db..664eacd 100644 --- a/frontend/src/render/details.ts +++ b/frontend/src/render/details.ts @@ -172,14 +172,18 @@ export function referralNote(service: Service): string { * short "added" note when the deploy added the device itself (addedId). The web app * shows the same step (app.js _showDeviceLink). */ -export function deviceLinkText(service: Service | undefined, values: Record, addedId = ""): string { +export function deviceLinkText(service: Service | undefined, values: Record, addedId = "", retryingId = ""): string { const link = service?.deviceLink; const id = (values[link?.idEnv || ""] || "").trim(); if (!link || !id) return ""; // The deploy added it through the provider's API (ProxyRack with its API key). if (addedId === id) return ["Added to your account", id].join("\n"); const url = (link.url || "").replace(/\$\{(\w+)\}/g, (_, key: string) => encodeURIComponent((values[key] || "").trim())); - return ["Link this device to your account", link.steps.trim(), id, url].filter(Boolean).join("\n"); + // The provider did not know the new device yet; the app keeps trying to add it. + const retrying = retryingId === id + ? "The provider does not know this new device yet. CashPilot keeps trying to add it for a day and tells you if it cannot." + : ""; + return ["Link this device to your account", retrying || link.steps.trim(), id, url].filter(Boolean).join("\n"); } /** One block of label/value rows. */ diff --git a/frontend/src/wails.d.ts b/frontend/src/wails.d.ts index a28c0ad..89eb5bd 100644 --- a/frontend/src/wails.d.ts +++ b/frontend/src/wails.d.ts @@ -300,6 +300,9 @@ export interface AppState { copies: Record | null; // Per slug, the device id the last deploy added to the user's account itself. addedDevices: Record | null; + // Per slug, the device id the provider did not know yet at deploy time; the app + // keeps trying to add it to the account for a day. + retryingDevices: Record | null; // Deployed services whose catalog marks state irreplaceable, and whether the // user has a backup of it (the amber card under Needs attention). stateBackups: StateBackupRow[] | null; diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 41a960a..21f8696 100755 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -1498,6 +1498,7 @@ export namespace main { scaling: Record; copies: Record>; addedDevices: Record; + retryingDevices: Record; stateBackups: statebackup.Row[]; drift: Record>; isolation?: runtime.FenceStatus; @@ -1527,6 +1528,7 @@ export namespace main { this.scaling = this.convertValues(source["scaling"], services.ScalingLimit, true); this.copies = this.convertValues(source["copies"], Array, true); this.addedDevices = source["addedDevices"]; + this.retryingDevices = source["retryingDevices"]; this.stateBackups = this.convertValues(source["stateBackups"], statebackup.Row); this.drift = source["drift"]; this.isolation = this.convertValues(source["isolation"], runtime.FenceStatus); diff --git a/internal/collectors/devices.go b/internal/collectors/devices.go index 1c3d656..1322895 100644 --- a/internal/collectors/devices.go +++ b/internal/collectors/devices.go @@ -136,39 +136,63 @@ func (r *Registry) checkProxyRackDevice(ctx context.Context, apiKey, uuid string } } +// Registration is what RegisterDevice made of a device. +type Registration int + +const ( + // RegisterFailed: not on the account, or this provider has no way to add it; + // the deploy output shows the manual step. + RegisterFailed Registration = iota + // RegisterAdded: the device is on the account. + RegisterAdded + // RegisterPending: the provider has not seen the device yet and says to try + // again later. A new ProxyRack UUID is unknown until its client has been + // connected for about five minutes; the app keeps retrying it. + RegisterPending +) + // RegisterDevice adds a just-deployed device to the user's account, where the // provider's API allows it and the credentials hold the key: today ProxyRack, with // /device/add, the call its own dashboard makes. A redeploy adds a UUID that is // already there, which the API refuses, so a refused add is followed by one status -// check: a known id is linked. Returns whether the device is on the account -// afterwards; false also means "this provider cannot do it", and the deploy output -// then shows the manual step. -func (r *Registry) RegisterDevice(ctx context.Context, slug string, credentials map[string]string) bool { +// check: a known id is linked. +func (r *Registry) RegisterDevice(ctx context.Context, slug string, credentials map[string]string) Registration { if slug != "proxyrack" { - return false + return RegisterFailed } apiKey := firstCredential(credentials, "API_KEY", "PROXYRACK_API_KEY", "api_key") uuid := strings.TrimSpace(credentials["UUID"]) if apiKey == "" || uuid == "" { - return false + return RegisterFailed } name := strings.TrimSpace(credentials["DEVICE_NAME"]) if name == "" || strings.Contains(name, "${") { name = "cashpilot-" + uuid[:min(8, len(uuid))] } r.noteDevice("proxyrack", uuid) + raw, status, _, err := r.doRaw(ctx, "POST", proxyRackAddURL, map[string]string{"device_id": uuid, "device_name": name}, proxyRackHeaders(apiKey), nil) var added struct { - Status string `json:"status"` + Status string `json:"status"` + Details struct { + DeviceID string `json:"device_id"` + } `json:"details"` + } + if err == nil { + _ = json.Unmarshal(raw, &added) // a body of another shape just matches nothing below } - status, err := r.doJSONStatus(ctx, "POST", proxyRackAddURL, map[string]string{"device_id": uuid, "device_name": name}, proxyRackHeaders(apiKey), nil, &added) linked := err == nil && status == http.StatusOK && added.Status == "success" + if !linked && err == nil && status == http.StatusBadRequest && + strings.Contains(strings.ToLower(added.Details.DeviceID), "try again") { + return RegisterPending + } if !linked { linked = r.proxyRackStatus(ctx, apiKey, uuid) == "linked" } - if linked { - r.recordDeviceCheck("proxyrack", uuid, true) + if !linked { + return RegisterFailed } - return linked + r.recordDeviceCheck("proxyrack", uuid, true) + return RegisterAdded } func proxyRackHeaders(apiKey string) map[string]string { diff --git a/internal/collectors/devices_test.go b/internal/collectors/devices_test.go index 4ac15b4..d7e0246 100644 --- a/internal/collectors/devices_test.go +++ b/internal/collectors/devices_test.go @@ -144,17 +144,23 @@ func TestRegisterDeviceAddsAProxyRackUUIDWithTheSavedKey(t *testing.T) { for _, tc := range []struct { name string routes map[string]stubResponse - want bool + want Registration }{ - {"added", map[string]stubResponse{proxyRackAdd: {body: `{"status":"success","message":"Device added successfully","data":{}}`}}, true}, + {"added", map[string]stubResponse{proxyRackAdd: {body: `{"status":"success","message":"Device added successfully","data":{}}`}}, RegisterAdded}, {"already on the account", map[string]stubResponse{ proxyRackAdd: {status: 400, body: `{"status":"error","error":"Invalid request parameters","details":{"device_id":"taken"}}`}, proxyRackStatus: {body: onlineBody}, - }, true}, + }, RegisterAdded}, {"rate limited", map[string]stubResponse{ proxyRackAdd: {status: 400, body: rateLimitedBody}, proxyRackStatus: {status: 400, body: rateLimitedBody}, - }, false}, + }, RegisterFailed}, + // A new UUID is unknown until its client has been connected for about five + // minutes; ProxyRack says so, and the app must retry instead of giving up. + {"not seen yet", map[string]stubResponse{ + proxyRackAdd: {status: 400, body: `{"status":"error","error":"Invalid request parameters","details":{"device_id":"Device not found. If you recently installed a device it may take 5 minutes before it is found in our system. Please try again in 5 minutes."}}`}, + proxyRackStatus: {status: 400, body: `{"status":"error","details":{"device_id":"The selected device id is invalid."}}`}, + }, RegisterPending}, } { t.Run(tc.name, func(t *testing.T) { stub := &stubTransport{routes: tc.routes} @@ -167,10 +173,10 @@ func TestRegisterDeviceAddsAProxyRackUUIDWithTheSavedKey(t *testing.T) { stub := &stubTransport{routes: map[string]stubResponse{proxyRackAdd: {body: `{"status":"success"}`}}} r := newTestRegistry(t, stub) - if r.RegisterDevice(context.Background(), "proxyrack", map[string]string{"UUID": testProxyRackUUID}) { + if r.RegisterDevice(context.Background(), "proxyrack", map[string]string{"UUID": testProxyRackUUID}) != RegisterFailed { t.Fatal("without an API key nothing can be added") } - if r.RegisterDevice(context.Background(), "earnapp", map[string]string{"EARNAPP_UUID": testEarnAppUUID, "EARNAPP_OAUTH_TOKEN": "t"}) { + if r.RegisterDevice(context.Background(), "earnapp", map[string]string{"EARNAPP_UUID": testEarnAppUUID, "EARNAPP_OAUTH_TOKEN": "t"}) != RegisterFailed { t.Fatal("EarnApp has no API to add a device; its step stays manual") } } diff --git a/internal/store/pending_device_links.go b/internal/store/pending_device_links.go new file mode 100644 index 0000000..a70c8c5 --- /dev/null +++ b/internal/store/pending_device_links.go @@ -0,0 +1,78 @@ +package store + +import "time" + +// A device registration the provider asked to retry later (a new ProxyRack UUID is +// unknown for about five minutes). The app retries it with backoff until the device +// is on the account, the device is gone, or a day has passed (gave_up = 1, and the +// user is told to add it by hand). Times are Unix seconds. +const pendingDeviceLinksDDL = ` + CREATE TABLE IF NOT EXISTS pending_device_links ( + slug TEXT NOT NULL, + device_id TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + first_at INTEGER NOT NULL, + next_at INTEGER NOT NULL, + gave_up INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (slug, device_id) + ); +` + +// PendingDeviceLink is one registration waiting to be retried. +type PendingDeviceLink struct { + Slug string + DeviceID string + Attempts int + FirstAt time.Time + NextAt time.Time + GaveUp bool +} + +// SavePendingDeviceLink remembers a registration to retry. A second save of a live +// one keeps its window and its earliest next try; one that had given up starts a +// new day of retries. +func (s *Store) SavePendingDeviceLink(slug, deviceID string, now, nextAt time.Time) error { + _, err := s.execWrite(` + INSERT INTO pending_device_links (slug, device_id, first_at, next_at) VALUES (?, ?, ?, ?) + ON CONFLICT(slug, device_id) DO UPDATE SET + first_at = CASE WHEN gave_up THEN excluded.first_at ELSE first_at END, + attempts = CASE WHEN gave_up THEN 0 ELSE attempts END, + next_at = CASE WHEN gave_up THEN excluded.next_at ELSE MIN(next_at, excluded.next_at) END, + gave_up = 0`, + slug, deviceID, now.Unix(), nextAt.Unix()) + return err +} + +// ListPendingDeviceLinks returns every stored registration, soonest retry first. +func (s *Store) ListPendingDeviceLinks() ([]PendingDeviceLink, error) { + rows, err := s.db.Query(`SELECT slug, device_id, attempts, first_at, next_at, gave_up FROM pending_device_links ORDER BY next_at`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PendingDeviceLink + for rows.Next() { + var p PendingDeviceLink + var first, next int64 + var gaveUp int + if err := rows.Scan(&p.Slug, &p.DeviceID, &p.Attempts, &first, &next, &gaveUp); err != nil { + return nil, err + } + p.FirstAt, p.NextAt, p.GaveUp = time.Unix(first, 0), time.Unix(next, 0), gaveUp != 0 + out = append(out, p) + } + return out, rows.Err() +} + +// UpdatePendingDeviceLink records the outcome of a retry that did not add the device. +func (s *Store) UpdatePendingDeviceLink(slug, deviceID string, attempts int, nextAt time.Time, gaveUp bool) error { + _, err := s.execWrite(`UPDATE pending_device_links SET attempts = ?, next_at = ?, gave_up = ? WHERE slug = ? AND device_id = ?`, + attempts, nextAt.Unix(), boolInt(gaveUp), slug, deviceID) + return err +} + +// DeletePendingDeviceLink forgets a registration: the device is on the account or gone. +func (s *Store) DeletePendingDeviceLink(slug, deviceID string) error { + _, err := s.execWrite(`DELETE FROM pending_device_links WHERE slug = ? AND device_id = ?`, slug, deviceID) + return err +} diff --git a/internal/store/pending_device_links_test.go b/internal/store/pending_device_links_test.go new file mode 100644 index 0000000..f8f8e58 --- /dev/null +++ b/internal/store/pending_device_links_test.go @@ -0,0 +1,78 @@ +package store + +import ( + "testing" + "time" +) + +func TestPendingDeviceLinks(t *testing.T) { + s := openTestStore(t) + t0 := time.Date(2026, 11, 8, 9, 0, 0, 0, time.UTC) + + if rows, err := s.ListPendingDeviceLinks(); err != nil || len(rows) != 0 { + t.Fatalf("a new store lists %+v, %v", rows, err) + } + if err := s.SavePendingDeviceLink("proxyrack", "AA", t0, t0.Add(6*time.Minute)); err != nil { + t.Fatal(err) + } + if err := s.SavePendingDeviceLink("proxyrack", "BB", t0, t0.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + rows, err := s.ListPendingDeviceLinks() + if err != nil || len(rows) != 2 || rows[0].DeviceID != "BB" || rows[1].DeviceID != "AA" { + t.Fatalf("want both rows, soonest retry first, got %+v, %v", rows, err) + } + if r := rows[1]; r.Slug != "proxyrack" || r.Attempts != 0 || r.GaveUp || !r.FirstAt.Equal(t0) || !r.NextAt.Equal(t0.Add(6*time.Minute)) { + t.Fatalf("row AA = %+v", r) + } + + // A second save of a live row keeps its window, its attempts and its earliest try. + if err := s.UpdatePendingDeviceLink("proxyrack", "AA", 2, t0.Add(time.Hour), false); err != nil { + t.Fatal(err) + } + later := t0.Add(2 * time.Hour) + if err := s.SavePendingDeviceLink("proxyrack", "AA", later, later.Add(6*time.Minute)); err != nil { + t.Fatal(err) + } + aa := find(t, s, "AA") + if aa.Attempts != 2 || !aa.FirstAt.Equal(t0) || !aa.NextAt.Equal(t0.Add(time.Hour)) || aa.GaveUp { + t.Fatalf("a live row was reset by a second save: %+v", aa) + } + + // One that gave up starts a new window. + if err := s.UpdatePendingDeviceLink("proxyrack", "AA", 9, t0.Add(time.Hour), true); err != nil { + t.Fatal(err) + } + if !find(t, s, "AA").GaveUp { + t.Fatal("UpdatePendingDeviceLink did not mark the row given up") + } + if err := s.SavePendingDeviceLink("proxyrack", "AA", later, later.Add(6*time.Minute)); err != nil { + t.Fatal(err) + } + aa = find(t, s, "AA") + if aa.Attempts != 0 || !aa.FirstAt.Equal(later) || !aa.NextAt.Equal(later.Add(6*time.Minute)) || aa.GaveUp { + t.Fatalf("a given-up row did not start a new window: %+v", aa) + } + + if err := s.DeletePendingDeviceLink("proxyrack", "AA"); err != nil { + t.Fatal(err) + } + if rows, _ := s.ListPendingDeviceLinks(); len(rows) != 1 || rows[0].DeviceID != "BB" { + t.Fatalf("after deleting AA, want only BB, got %+v", rows) + } +} + +func find(t *testing.T, s *Store, id string) PendingDeviceLink { + t.Helper() + rows, err := s.ListPendingDeviceLinks() + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if r.DeviceID == id { + return r + } + } + t.Fatalf("no row for %s in %+v", id, rows) + return PendingDeviceLink{} +} diff --git a/internal/store/store.go b/internal/store/store.go index 3039d15..df85741 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1009,6 +1009,9 @@ func (s *Store) migrate() error { if _, err = s.execWrite(proxiesDDL); err != nil { return err } + if _, err = s.execWrite(pendingDeviceLinksDDL); err != nil { + return err + } if err := s.migrateUsage(); err != nil { return err }