fix(hardcover): stop asking with a rejected API token (#3101)

changelog.d/3101-rejected-hardcover-token-hold.md

@@ -0,0 +1,2 @@
+### Fixed
+- **A rejected Hardcover token no longer costs a request per row** (#3101). When Hardcover refuses the saved API token (a 401, or a 403 that is an authentication failure), Bindery now stops asking with that token. A bulk import, discovery sweep, catalogue sync, list sync or scheduled refresh stops at the first rejection and says "Hardcover rejected the API token", instead of sending one doomed request for every row. Settings shows a notice, saving a new token lifts the pause immediately, **Test connection** still reaches Hardcover so you can retest after fixing the token, and one probe request an hour lets a restored token recover by itself. Server errors and queries Hardcover refuses for other reasons do not pause anything.

cmd/bindery/discovery_wiring.go

@@ -28,13 +28,12 @@ func newAuthorDiscoverer(discover authorDiscoverFunc, bulkRunning func() bool) s
 	return scheduler.AuthorDiscovererFuncs{
 		Discover: func(ctx context.Context, author *models.Author) scheduler.DiscoveryOutcome {
 			created, err := discover(ctx, author)
-			var daily *metadata.DailyQuotaError
 			out := scheduler.DiscoveryOutcome{
 				Created: created,
 				Err:     err,
 				// hardcover.ErrRateLimited is checked too: the bare sentinel does
 				// not carry the shared mark, only its throttle errors do.
-				Backoff: errors.As(err, &daily) || errors.Is(err, metadata.ErrRateLimited) || errors.Is(err, hardcover.ErrRateLimited),
+				Backoff: metadata.IsProviderHold(err) || errors.Is(err, metadata.ErrRateLimited) || errors.Is(err, hardcover.ErrRateLimited),
 				Busy:    errors.Is(err, api.ErrAuthorSyncRunning),
 			}
 			out.Unavailable = !out.Backoff && !out.Busy && metadata.IsProviderUnavailable(err)

cmd/bindery/discovery_wiring_test.go

@@ -28,6 +28,8 @@ func TestNewAuthorDiscoverer_MapsErrors(t *testing.T) {
 	}{
 		{name: "success", created: 3},
 		{name: "daily quota", err: &metadata.DailyQuotaError{}, wantBackoff: true},
+		{name: "rejected token", err: &metadata.TokenRejectedError{Provider: "Hardcover"}, wantBackoff: true},
+		{name: "rejected token, wrapped", err: fmt.Errorf("author works: %w", &metadata.TokenRejectedError{Provider: "Hardcover"}), wantBackoff: true},
 		{name: "ordinary error", err: errors.New("provider 500")},
 		{name: "hardcover rate limit, wrapped", err: fmt.Errorf("author works: %w", hardcover.ErrRateLimited), wantBackoff: true},
 		{name: "openlibrary rate limit, wrapped", err: fmt.Errorf("author works: %w", openlibrary.ErrRateLimited), wantBackoff: true},

cmd/bindery/hardcover_status.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"errors"
+	"log/slog"
+	"time"
+
+	"github.com/vavallee/bindery/internal/metadata"
+)
+
+// hardcoverHoldStatus turns the result of Hardcover's CheckQuota into the two
+// fields GET /api/v1/system/status reports: the end of a daily quota pause (an
+// RFC 3339 time, empty when there is none) and whether the saved token is
+// currently refused. A rejected token outranks a quota pause, as it does in
+// CheckQuota itself, so the Settings banner never promises a reset that will
+// not help (#3101).
+func hardcoverHoldStatus(err error) (pausedUntil string, tokenRejected bool) {
+	var rejected *metadata.TokenRejectedError
+	var exhausted *metadata.DailyQuotaError
+	switch {
+	case errors.As(err, &rejected):
+		return "", true
+	case errors.As(err, &exhausted):
+		return exhausted.ResetAt.UTC().Format(time.RFC3339), false
+	case err != nil:
+		slog.Warn("read Hardcover pause status", "error", err)
+	}
+	return "", false
+}

cmd/bindery/hardcover_status_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/vavallee/bindery/internal/metadata"
+)
+
+func TestHardcoverHoldStatus(t *testing.T) {
+	reset := time.Date(2026, 10, 12, 0, 0, 0, 0, time.UTC)
+	for _, tc := range []struct {
+		name      string
+		err       error
+		wantUntil string
+		wantRej   bool
+	}{
+		{"no hold", nil, "", false},
+		{"daily quota", &metadata.DailyQuotaError{ResetAt: reset}, "2026-10-12T00:00:00Z", false},
+		{"rejected token", &metadata.TokenRejectedError{Provider: "Hardcover"}, "", true},
+		{"rejected token joined with a write failure", errors.Join(&metadata.TokenRejectedError{Provider: "Hardcover"}, errors.New("disk full")), "", true},
+		{"wrapped daily quota", fmt.Errorf("check: %w", &metadata.DailyQuotaError{ResetAt: reset}), "2026-10-12T00:00:00Z", false},
+		{"unrelated error", errors.New("settings unreadable"), "", false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			until, rejected := hardcoverHoldStatus(tc.err)
+			if until != tc.wantUntil || rejected != tc.wantRej {
+				t.Fatalf("got (%q, %v), want (%q, %v)", until, rejected, tc.wantUntil, tc.wantRej)
+			}
+		})
+	}
+}

cmd/bindery/main.go

@@ -955,13 +955,7 @@ func main() {
 			w.Header().Set("Content-Type", "application/json")
 			cacheBytes, _ := imageProxyHandler.CacheSize()
 			hardcoverState := api.HardcoverFeatureStateFor(r.Context(), settingsRepo, cfg.EnhancedHardcoverAPI)
-			var pausedUntil string
-			var exhausted *metadata.DailyQuotaError
-			if err := hcClient.CheckQuota(r.Context()); errors.As(err, &exhausted) {
-				pausedUntil = exhausted.ResetAt.UTC().Format(time.RFC3339)
-			} else if err != nil {
-				slog.Warn("read Hardcover pause status", "error", err)
-			}
+			pausedUntil, tokenRejected := hardcoverHoldStatus(hcClient.CheckQuota(r.Context()))
 			_ = json.NewEncoder(w).Encode(struct {
 				Version   string `json:"version"`
 				Commit    string `json:"commit"`
@@ -975,6 +969,7 @@ func main() {
 				EnhancedHardcoverAPI            bool   `json:"enhancedHardcoverApi"`
 				HardcoverTokenConfigured        bool   `json:"hardcoverTokenConfigured"`
 				HardcoverPausedUntil            string `json:"hardcoverPausedUntil,omitempty"`
+				HardcoverTokenRejected          bool   `json:"hardcoverTokenRejected,omitempty"`
 				EnhancedHardcoverDisabledReason string `json:"enhancedHardcoverDisabledReason,omitempty"`
 			}{
 				Version:                         version,
@@ -985,6 +980,7 @@ func main() {
 				EnhancedHardcoverAPI:            hardcoverState.EnhancedHardcoverAPI,
 				HardcoverTokenConfigured:        hardcoverState.HardcoverTokenConfigured,
 				HardcoverPausedUntil:            pausedUntil,
+				HardcoverTokenRejected:          tokenRejected,
 				EnhancedHardcoverDisabledReason: hardcoverState.EnhancedHardcoverDisabledReason,
 			})
 		})

docs/API.md

@@ -765,7 +765,7 @@ POST   /api/v1/backup                             snapshot the SQLite database (
 GET    /api/v1/backup                             list stored backups (admin)
 DELETE /api/v1/backup/{filename}                  delete one backup (admin)
 POST   /api/v1/backup/{filename}/restore          stage a backup for the next restart (admin, X-Confirm-Restore: true)
-GET    /api/v1/system/status                      version, commit, build date, newest published release, image cache size, Hardcover feature state
+GET    /api/v1/system/status                      version, commit, build date, newest published release, image cache size, Hardcover feature state; `hardcoverPausedUntil` while the daily quota holds, `hardcoverTokenRejected` while Hardcover has refused the saved token
 POST   /api/v1/library/scan                       start a library scan in the background (202; {"queued": true} when one is already running)
 GET    /api/v1/library/duplicate-candidates      read-only duplicate title groups across every author, paginated (#2999)
 GET    /api/v1/library/scan/status                summary of the last library scan, paths included (admin)

docs/DEPLOYMENT.md

@@ -746,6 +746,30 @@ Scheduled jobs can run again on their next scheduled pass. This does not add
 automatic edition-hydration recovery or request accounting.
 Different tokens have separate holds, even if they belong to the same account.
 
+### Hardcover rejected token pause
+
+When Hardcover answers a request with a JSON 401 (a 401 page that is HTML or plain
+text came from a proxy or CDN and does not count), or with a 403 whose body is an
+authentication failure (`invalid_token`, `unauthorized`, `invalid_grant` or
+"Unable to verify token"), Bindery stops sending requests with that token. A
+bulk import, a discovery sweep, a catalogue sync, a list sync and the scheduled
+refresh stop at the first rejection instead of asking once per row, and the
+import result carries "Hardcover rejected the API token". Server errors (5xx),
+a 403 that only describes one query (`insufficient_scope`, `access_denied`,
+`invalid_request`) and GraphQL errors inside a 200 response do not start a pause.
+
+The pause is keyed to the token in the same way as the daily quota hold, so it
+survives a restart and saving a different token in **Settings → API Keys** lifts
+it at once. It also ends after one hour without a save: the next request is
+sent as a probe and either clears the hold (Hardcover accepted the token) or
+renews it. That is one request an hour for a token that stays bad. **Test
+connection** always sends its request, so a token fixed on Hardcover's side can
+be retested immediately, and a success lifts the pause. The pause applies to
+every Hardcover call, including when Hardcover is only an enricher, a cover or
+ISBN source or the list syncer. The hold is stored with the quota holds as a
+private setting and is hidden from generic settings endpoints; a corrupt row is
+logged and ignored.
+
 ### Enhanced Hardcover series data deployment note
 
 **Schema:** enhanced series data uses migration `035`, which creates `series_hardcover_links` and backfills links for existing series whose foreign ID already points at Hardcover. Take a normal SQLite backup before upgrading, then let Bindery apply the migration on startup.

internal/abs/daily_quota_test.go

@@ -13,11 +13,16 @@ type heldProvider struct {
 	metadata.Provider
 	checks    int
 	stopAfter int
+	// rejected makes the hold a rejected token instead of the daily quota.
+	rejected bool
 }
 
 func (p *heldProvider) CheckQuota(context.Context) error {
 	p.checks++
 	if p.checks > p.stopAfter {
+		if p.rejected {
+			return &metadata.TokenRejectedError{Provider: "Hardcover"}
+		}
 		return &metadata.DailyQuotaError{ResetAt: time.Date(2026, 9, 19, 1, 0, 0, 0, time.UTC)}
 	}
 	return nil
@@ -46,3 +51,22 @@ func TestDailyQuotaStopsABSBeforeCheckpointingItem(t *testing.T) {
 		})
 	}
 }
+
+func TestTokenRejectedStopsABSBeforeCheckpointingItem(t *testing.T) {
+	importer, _, _, _, _, _, _, _, _, _ := newABSImporterFixture(t)
+	importer.WithMetadata(metadata.NewAggregator(&heldProvider{stopAfter: 1, rejected: true}))
+	accepted := 0
+	importer.enumerateFn = func(ctx context.Context, _ string, fn func(context.Context, NormalizedLibraryItem) error) (EnumerationStats, error) {
+		for range 3 {
+			if err := fn(ctx, sampleABSItem()); err != nil {
+				return EnumerationStats{}, err
+			}
+			accepted++
+		}
+		return EnumerationStats{}, nil
+	}
+	_, err := importer.Run(context.Background(), ImportConfig{SourceID: DefaultSourceID, BaseURL: "https://abs.example.com", APIKey: "example-only", LibraryID: "lib-books", Enabled: true, DryRun: true})
+	if err == nil || !strings.Contains(err.Error(), "Hardcover rejected the API token") || accepted != 0 {
+		t.Fatalf("accepted %d err %v", accepted, err)
+	}
+}

internal/api/rejected_token_test.go

@@ -0,0 +1,114 @@
+package api
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/vavallee/bindery/internal/db"
+	"github.com/vavallee/bindery/internal/httpsec"
+	"github.com/vavallee/bindery/internal/metadata"
+	"github.com/vavallee/bindery/internal/metadata/hardcover"
+)
+
+// rejectedHoldRow stores a live rejected token hold for token under a known
+// install secret, as a restart would find it.
+func rejectedHoldRow(t *testing.T, settings *db.SettingsRepo, token string) {
+	t.Helper()
+	ctx := context.Background()
+	secret := strings.Repeat("ab", 32)
+	if err := settings.Set(ctx, "auth.hardcover_daily_hold_secret", secret); err != nil {
+		t.Fatal(err)
+	}
+	key, _ := hex.DecodeString(secret)
+	mac := hmac.New(sha256.New, key)
+	mac.Write([]byte(token))
+	until := time.Now().UTC().Add(time.Hour).Format(time.RFC3339)
+	if err := settings.Set(ctx, "auth.hardcover_rejected_holds", fmt.Sprintf(`{"%x":%q}`, mac.Sum(nil), until)); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRejectedTokenHoldDoesNotBlockTestConnection(t *testing.T) {
+	h, settings, ctx := settingsFixture(t)
+	token := "example-only"
+	if err := settings.Set(ctx, SettingHardcoverAPIToken, token); err != nil {
+		t.Fatal(err)
+	}
+	rejectedHoldRow(t, settings, token)
+	hold := hardcover.NewDailyQuota(settings)
+	h.WithDailyQuota(hold)
+
+	// The same held token is refused for ordinary calls without a request.
+	var rejected *metadata.TokenRejectedError
+	if err := hardcover.New().WithDailyQuota(hold).WithToken(token).CheckQuota(ctx); !errors.As(err, &rejected) {
+		t.Fatalf("hold not live: %v", err)
+	}
+	// The test button must get past the hold. An outbound proxy that refuses
+	// every tunnel stands in for Hardcover and counts the attempts, so nothing
+	// leaves the machine.
+	t.Cleanup(func() { _, _ = httpsec.ConfigureOutboundProxy("", "", true) })
+	var attempts atomic.Int32
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		attempts.Add(1)
+		w.WriteHeader(http.StatusBadGateway)
+	}))
+	defer proxy.Close()
+	if _, err := httpsec.ConfigureOutboundProxy(proxy.URL, "", true); err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	h.TestHardcover(rec, httptest.NewRequest(http.MethodPost, "/api/v1/hardcover/test", nil))
+	if body := rec.Body.String(); attempts.Load() == 0 || strings.Contains(body, "rejected the API token") {
+		t.Fatalf("test connection was blocked by the rejected hold (attempts %d): %s", attempts.Load(), body)
+	}
+}
+
+func TestSavingNewTokenLiftsRejectedHold(t *testing.T) {
+	_, settings, ctx := settingsFixture(t)
+	rejectedHoldRow(t, settings, "old-token")
+	if err := settings.Set(ctx, SettingHardcoverAPIToken, "old-token"); err != nil {
+		t.Fatal(err)
+	}
+	// Wired as in cmd/bindery: the token is read per call from settings.
+	client := hardcover.New().WithDailyQuota(hardcover.NewDailyQuota(settings)).WithTokenSource(func(ctx context.Context) string {
+		return GetHardcoverAPIToken(ctx, settings)
+	})
+	var rejected *metadata.TokenRejectedError
+	if err := client.CheckQuota(ctx); !errors.As(err, &rejected) {
+		t.Fatalf("hold not live: %v", err)
+	}
+	if err := settings.Set(ctx, SettingHardcoverAPIToken, "new-token"); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.CheckQuota(ctx); err != nil {
+		t.Fatalf("new token still held: %v", err)
+	}
+}
+
+func TestRejectedTokenHoldHiddenFromAdminSettings(t *testing.T) {
+	h, settings, ctx := settingsFixture(t)
+	key := "auth.hardcover_rejected_holds"
+	if err := settings.Set(ctx, key, `{"example-fingerprint":"2026-09-19T01:00:00Z"}`); err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	h.List(rec, adminReq(httptest.NewRequest(http.MethodGet, "/api/v1/setting", nil)))
+	if rec.Code != http.StatusOK || strings.Contains(rec.Body.String(), key) || strings.Contains(rec.Body.String(), "example-fingerprint") {
+		t.Fatalf("list: %d %s", rec.Code, rec.Body.String())
+	}
+	rec = httptest.NewRecorder()
+	h.Get(rec, adminReq(withKey(httptest.NewRequest(http.MethodGet, "/api/v1/setting/"+key, nil), key)))
+	if rec.Code != http.StatusNotFound {
+		t.Fatalf("get: %d %s", rec.Code, rec.Body.String())
+	}
+}

internal/api/settings_handler.go

@@ -314,6 +314,7 @@ func isSecretSetting(key string) bool {
 	// 1. Explicit one-offs that don't match the patterns below.
 	switch key {
 	case "auth.hardcover_daily_holds",
+		"auth.hardcover_rejected_holds",
 		SettingAuthAPIKey,
 		SettingAuthSessionSecret,
 		SettingAuthSessionSecretPrevious,
@@ -679,6 +680,10 @@ func (h *SettingsHandler) TestHardcover(w http.ResponseWriter, r *http.Request)
 
 	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
 	defer cancel()
+	// A user retesting after fixing the token on Hardcover's side must reach
+	// Hardcover even while a rejected token hold is live (#3101). A daily quota
+	// hold still answers without a request.
+	ctx = hardcover.WithRejectedTokenProbe(ctx)
 
 	client := hardcover.New().WithDailyQuota(h.dailyQuota).WithToken(token)
 	series, err := client.SearchSeries(ctx, "Dune", 3)

internal/hardcoverlistsyncer/rejected_token_test.go

@@ -0,0 +1,45 @@
+package hardcoverlistsyncer
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/vavallee/bindery/internal/db"
+	"github.com/vavallee/bindery/internal/metadata"
+	"github.com/vavallee/bindery/internal/metadata/hardcover"
+)
+
+func TestRejectedTokenListSyncStopsBeforeFetch(t *testing.T) {
+	database, err := db.OpenMemory()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer database.Close()
+	ctx := context.Background()
+	settings := db.NewSettingsRepo(database)
+	token := "example-only"
+	secret := strings.Repeat("ab", 32)
+	if err := settings.Set(ctx, "auth.hardcover_daily_hold_secret", secret); err != nil {
+		t.Fatal(err)
+	}
+	key, _ := hex.DecodeString(secret)
+	mac := hmac.New(sha256.New, key)
+	mac.Write([]byte(token))
+	if err := settings.Set(ctx, "auth.hardcover_rejected_holds", fmt.Sprintf(`{"%x":%q}`, mac.Sum(nil), time.Now().Add(time.Hour).UTC().Format(time.RFC3339))); err != nil {
+		t.Fatal(err)
+	}
+	s := New(db.NewImportListRepo(database), db.NewAuthorRepo(database), db.NewBookRepo(database)).WithDailyQuota(hardcover.NewDailyQuota(settings))
+	il := testImportList("Held", "hardcover", true)
+	il.APIKey = token
+	var rejected *metadata.TokenRejectedError
+	if err := s.syncList(ctx, il); !errors.As(err, &rejected) {
+		t.Fatalf("factory hold: %v", err)
+	}
+}

internal/metadata/aggregator.go

@@ -920,8 +920,7 @@ func (a *Aggregator) GetBookByISBNWithOutcome(ctx context.Context, isbn string)
 		if err != nil {
 			// A held primary stops the lookup; a held enricher is just a failed
 			// provider, so the primary's answer and later fallbacks still count.
-			var daily *DailyQuotaError
-			if idx == 0 && errors.As(err, &daily) {
+			if idx == 0 && IsProviderHold(err) {
 				return nil, newSearchOutcome(primaryName, []string{normalizedProviderName(providerName(provider))}, answered, err), err
 			}
 			if errors.Is(err, ErrProviderNotConfigured) {