diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7e003b6b7..c9c7f8b54 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -95,6 +95,12 @@ jobs: go test ./... echo "--- Decrypt CLI tests passed ---" + - name: Test shared channel registry + schema packages + run: | + set -e -o pipefail + (cd internal/channelregistry && go test ./...) + (cd internal/dbschema && go test ./...) + - name: Verify Dockerfile COPY invariants (issue #1316) run: bash scripts/check-dockerfile-internal-pkgs.sh @@ -126,6 +132,9 @@ jobs: node test-channel-decrypt-insecure-context.js node test-live-region-filter.js node test-issue-2052-touch-target-css.js + node test-channel-proposals.js + node test-packet-detail-channel-xss.js + node test-channels-name-escaping.js node test-issue-1136-observer-iata-map.js node test-channel-qr.js node test-channel-qr-wiring.js @@ -309,6 +318,12 @@ jobs: go build -o ../../corescope-migrate . echo "Go migrate tool built successfully" + - name: Build Go ingestor (channel proposals E2E) + run: | + cd cmd/ingestor + go build -o ../../corescope-ingestor . + echo "Go ingestor built successfully" + - name: Install npm dependencies run: npm ci --production=false @@ -496,6 +511,8 @@ jobs: BASE_URL=http://localhost:13581 node test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1367-channels-chat-app-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-2052-touch-target-e2e.js 2>&1 | tee -a e2e-output.txt + # Starts its own ingestor + server on a temp copy of the fixture. + CHROMIUM_REQUIRE=1 CORESCOPE_SERVER_BIN=./corescope-server CORESCOPE_INGESTOR_BIN=./corescope-ingestor FIXTURE_DB=test-fixtures/e2e-fixture.db node test-channel-proposals-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1329-map-controls-accordion-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt diff --git a/Dockerfile b/Dockerfile index dc8a2165a..daaaf73de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ COPY internal/lora/ ../../internal/lora/ COPY internal/regions/ ../../internal/regions/ +COPY internal/channelregistry/ ../../internal/channelregistry/ RUN go mod download COPY cmd/server/ ./ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ @@ -42,6 +43,7 @@ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ COPY internal/regions/ ../../internal/regions/ +COPY internal/channelregistry/ ../../internal/channelregistry/ RUN go mod download COPY cmd/ingestor/ ./ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ diff --git a/cmd/ingestor/channel_proposals.go b/cmd/ingestor/channel_proposals.go new file mode 100644 index 000000000..78a11eea9 --- /dev/null +++ b/cmd/ingestor/channel_proposals.go @@ -0,0 +1,401 @@ +package main + +// Shared channel proposals — the writer side. +// +// The read-only server validates public suggestions and admin decisions and +// writes them as command files into the channelregistry queue next to the +// database. This file is where they become rows: the ingestor applies each +// command in a transaction, writes a result file the server reads back, and +// adds newly approved channels to the live channel keys so their traffic is +// decrypted from then on. +// +// Every command is idempotent, because a crash can land between the commit +// and the result write (the command is then applied again on the next tick): +// - submit: the new proposal's id IS the request id, so a replay finds its +// own row and reports it instead of inserting twice. A different request +// for an existing name is a duplicate and reports the existing proposal. +// - approve/reject: repeating the decision that is already stored reports +// it unchanged; only a *conflicting* repeat (reject after approve) fails. + +import ( + "context" + "database/sql" + "errors" + "log" + "time" + + "github.com/meshcore-analyzer/channelregistry" +) + +// User-facing outcomes. Their text is returned verbatim in the request status. +var ( + errTooManyPending = errors.New("too many suggestions are waiting for review, please try again later") + errTooManyApproved = errors.New("the limit of shared channels has been reached") + errProposalNotFound = errors.New("suggestion not found") + errRequestExpired = errors.New("the request expired before it could be processed") + errProposalStorage = errors.New("the suggestion could not be saved") +) + +// errProposalConflict reports a decision that contradicts the stored one. +// source is the status the decision needed (pending for approve/reject, +// approved for revoke); status is the one actually stored. +type errProposalConflict struct{ source, status string } + +func (e errProposalConflict) Error() string { + if e.source == channelregistry.StatusApproved { + // Revoke: "already pending" would read as if revoking had worked. + return "suggestion is not approved (it is " + e.status + ")" + } + return "suggestion was already " + e.status +} + +const proposalColumns = `id, name, status, created_at, reviewed_at` + +func scanProposalRow(row *sql.Row) (channelregistry.Proposal, bool, error) { + var p channelregistry.Proposal + var reviewed sql.NullInt64 + err := row.Scan(&p.ID, &p.Name, &p.Status, &p.CreatedAt, &reviewed) + if err == sql.ErrNoRows { + return p, false, nil + } + if err != nil { + return p, false, err + } + if reviewed.Valid { + v := reviewed.Int64 + p.ReviewedAt = &v + } + return p, true, nil +} + +// submitChannelProposal stores a new pending proposal, or reports the existing +// one for a replayed request or an already-proposed name. A rejected name +// therefore stays blocked — re-suggesting it reports the rejection — until +// retention (RetentionDays after the review) deletes the row; that keeps a +// rejected name from being pushed back into the review queue over and over. +// A name that was previously approved and then revoked is free to re-propose: +// the existing row (its id, and so its identity, is kept — UNIQUE(name) means +// it cannot become a second row) is resurrected to pending rather than left +// revoked forever or auto-approved. +func (s *Store) submitChannelProposal(ctx context.Context, cmd channelregistry.Command, maxPending int, nowMs int64) (channelregistry.Proposal, error) { + // Never trust the queue file: validate again with the shared rules. + name, err := channelregistry.NormalizeName(cmd.Name) + if err != nil { + return channelregistry.Proposal{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return channelregistry.Proposal{}, err + } + defer tx.Rollback() + + // Prefer our own row (replay) over another request's row (duplicate). + existing, found, err := scanProposalRow(tx.QueryRowContext(ctx, + `SELECT `+proposalColumns+` FROM channel_proposals WHERE id = ? OR name = ? + ORDER BY (id = ?) DESC LIMIT 1`, cmd.RequestID, name, cmd.RequestID)) + if err != nil { + return channelregistry.Proposal{}, err + } + if found && existing.Status != channelregistry.StatusRevoked { + return existing, nil + } + var pending int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM channel_proposals WHERE status = 'pending'`).Scan(&pending); err != nil { + return channelregistry.Proposal{}, err + } + if pending >= maxPending { + return channelregistry.Proposal{}, errTooManyPending + } + created := cmd.CreatedAt + if created <= 0 { + created = nowMs + } + if found { + // existing.Status == StatusRevoked here (the only other reachable + // branch above already returned). Resurrect the same row instead of + // inserting a new one, using its ORIGINAL id, not cmd.RequestID: a + // later replay of THIS exact resubmission command looks the row up + // by name (its id will not match cmd.RequestID), finds it already + // 'pending', and falls into the "return existing unchanged" branch + // above — which is what makes resubmission idempotent across a + // crash-and-retry, the same way plain submit is idempotent via id. + res, err := tx.ExecContext(ctx, + `UPDATE channel_proposals SET status = 'pending', created_at = ?, reviewed_at = NULL WHERE id = ? AND status = 'revoked'`, + created, existing.ID) + if err != nil { + return channelregistry.Proposal{}, err + } + if n, err := res.RowsAffected(); err != nil { + return channelregistry.Proposal{}, err + } else if n == 0 { + // Guard tripped: something else changed this row between the + // SELECT and the UPDATE above. Report it rather than fabricate a + // result — the next tick's replay (if any) resolves it cleanly. + return channelregistry.Proposal{}, errProposalStorage + } + if err := tx.Commit(); err != nil { + return channelregistry.Proposal{}, err + } + return channelregistry.Proposal{ID: existing.ID, Name: name, Status: channelregistry.StatusPending, CreatedAt: created}, nil + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO channel_proposals (id, name, status, created_at) VALUES (?, ?, 'pending', ?)`, + cmd.RequestID, name, created); err != nil { + return channelregistry.Proposal{}, err + } + if err := tx.Commit(); err != nil { + return channelregistry.Proposal{}, err + } + return channelregistry.Proposal{ID: cmd.RequestID, Name: name, Status: channelregistry.StatusPending, CreatedAt: created}, nil +} + +// reviewChannelProposal moves a proposal currently in status source to +// status target: approve/reject move pending → approved/rejected, revoke +// moves approved → revoked. Repeating the same decision (source already == +// target) is idempotent; anything else that isn't source is a conflict. +func (s *Store) reviewChannelProposal(ctx context.Context, id, source, target string, maxApproved int, nowMs int64) (channelregistry.Proposal, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return channelregistry.Proposal{}, err + } + defer tx.Rollback() + p, found, err := scanProposalRow(tx.QueryRowContext(ctx, + `SELECT `+proposalColumns+` FROM channel_proposals WHERE id = ?`, id)) + if err != nil { + return channelregistry.Proposal{}, err + } + if !found { + return channelregistry.Proposal{}, errProposalNotFound + } + if p.Status == target { + return p, nil // repeated decision: nothing to do + } + if p.Status != source { + return p, errProposalConflict{source: source, status: p.Status} + } + // Capacity is only consumed by approving; revoking (target == revoked) + // frees a slot instead, so this check naturally does not apply to it. + if target == channelregistry.StatusApproved { + var approved int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM channel_proposals WHERE status = 'approved'`).Scan(&approved); err != nil { + return channelregistry.Proposal{}, err + } + if approved >= maxApproved { + return p, errTooManyApproved + } + } + // reviewed_at is reused as "last status-change timestamp" for revoke too, + // same as it already is for approve/reject. + if _, err := tx.ExecContext(ctx, + `UPDATE channel_proposals SET status = ?, reviewed_at = ? WHERE id = ? AND status = ?`, + target, nowMs, id, source); err != nil { + return channelregistry.Proposal{}, err + } + if err := tx.Commit(); err != nil { + return channelregistry.Proposal{}, err + } + p.Status = target + p.ReviewedAt = &nowMs + return p, nil +} + +// PruneChannelProposals deletes rejected/revoked proposals reviewed before +// cutoffMs and pending ones submitted before it. Approved channels are kept: +// they are bounded by maxApproved and must stay active. A revoked row is not +// deleted at revoke time — only its status changes, preserving reviewed_at +// as the audit timestamp — but it is not kept forever either: retention +// removes it exactly like a rejected row once it is old enough. +func (s *Store) PruneChannelProposals(ctx context.Context, cutoffMs int64) (int64, error) { + res, err := s.db.ExecContext(ctx, + `DELETE FROM channel_proposals + WHERE (status IN ('rejected', 'revoked') AND reviewed_at < ?) + OR (status = 'pending' AND created_at < ?)`, cutoffMs, cutoffMs) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// channelProposalRunner drains the queue and keeps the live keys in step. +type channelProposalRunner struct { + store *Store + queue *channelregistry.Queue + keys *hotKeys + limits channelregistry.Limits + now func() time.Time + + // publishedBaseGen is the hotKeys base generation last written to the + // builtin names file; 0 means never written. + publishedBaseGen uint64 +} + +func newChannelProposalRunner(store *Store, keys *hotKeys, cfg *channelregistry.Config) *channelProposalRunner { + return &channelProposalRunner{ + store: store, + queue: channelregistry.NewQueue(channelregistry.QueueDir(store.path)), + keys: keys, + limits: cfg.Limits(), + now: time.Now, + } +} + +// LoadApproved adds every approved channel to the live keys. Called at +// startup, before MQTT traffic is handled. +func (r *channelProposalRunner) LoadApproved(ctx context.Context) (int, error) { + names, err := channelregistry.ListApprovedNames(ctx, r.store.db, channelregistry.MaxListed) + if err != nil { + return 0, err + } + return r.keys.AddApproved(names...), nil +} + +// RunOnce publishes the builtin names when the config-derived key layer has +// changed, then applies every waiting command in submission order. +func (r *channelProposalRunner) RunOnce(ctx context.Context) { + r.publishBuiltinNames() + cmds, err := r.queue.Pending() + if err != nil { + log.Printf("[channel-proposals] list queue failed: %v", err) + return + } + for _, qc := range cmds { + if qc.Err != nil { + log.Printf("[channel-proposals] discarding unreadable command %s: %v", qc.Path, qc.Err) + if err := r.queue.Discard(qc.Path); err != nil { + log.Printf("[channel-proposals] discard %s failed: %v", qc.Path, err) + } + continue + } + res := r.apply(ctx, qc.Command) + // Keys first: the row is committed, so the channel is approved even + // if writing the result fails below (it is retried next tick). + if qc.Command.Op == channelregistry.OpApprove && res.Status == channelregistry.RequestApproved && res.Proposal != nil { + if r.keys.AddApproved(res.Proposal.Name) > 0 { + log.Printf("[channel-proposals] approved %q — added to channel keys", res.Proposal.Name) + } + } + if qc.Command.Op == channelregistry.OpRevoke && res.Status == channelregistry.RequestRevoked && res.Proposal != nil { + if r.keys.RemoveApproved(res.Proposal.Name) { + log.Printf("[channel-proposals] revoked %q — removed from channel keys", res.Proposal.Name) + } + } + if err := r.queue.Complete(res); err != nil { + log.Printf("[channel-proposals] write result %s failed: %v", res.RequestID, err) + } + } +} + +// publishBuiltinNames writes the hashtag names of the base key layer (built-in, +// rainbow table, hashChannels, channelKeys) for the server, once at startup +// and again after each reload. Those names decrypt whether or not they are +// approved, because base wins over approved in hotKeys. O(base) and only when +// the base generation changed, never per tick. +func (r *channelProposalRunner) publishBuiltinNames() { + names, gen := r.keys.BaseNames() + if gen == r.publishedBaseGen { + return + } + if err := r.queue.WriteBuiltinNames(names); err != nil { + log.Printf("[channel-proposals] publishing built-in channel names failed: %v", err) + return + } + r.publishedBaseGen = gen +} + +func (r *channelProposalRunner) apply(ctx context.Context, cmd channelregistry.Command) channelregistry.Result { + now := r.now() + nowMs := now.UnixMilli() + res := channelregistry.Result{RequestID: cmd.RequestID, CompletedAt: nowMs} + if cmd.CreatedAt > 0 && nowMs-cmd.CreatedAt > r.limits.RequestTTL.Milliseconds() { + res.Status, res.Error = channelregistry.RequestError, errRequestExpired.Error() + return res + } + var ( + p channelregistry.Proposal + err error + ) + switch cmd.Op { + case channelregistry.OpSubmit: + p, err = r.store.submitChannelProposal(ctx, cmd, r.limits.MaxPending, nowMs) + case channelregistry.OpApprove: + p, err = r.store.reviewChannelProposal(ctx, cmd.ProposalID, channelregistry.StatusPending, channelregistry.StatusApproved, r.limits.MaxApproved, nowMs) + case channelregistry.OpReject: + p, err = r.store.reviewChannelProposal(ctx, cmd.ProposalID, channelregistry.StatusPending, channelregistry.StatusRejected, r.limits.MaxApproved, nowMs) + case channelregistry.OpRevoke: + p, err = r.store.reviewChannelProposal(ctx, cmd.ProposalID, channelregistry.StatusApproved, channelregistry.StatusRevoked, r.limits.MaxApproved, nowMs) + default: + err = channelregistry.ErrInvalidCommand + } + if err != nil { + res.Status, res.Error = channelregistry.RequestError, userFacingProposalError(cmd, err) + if p.ID != "" { + res.Proposal = &p + } + return res + } + res.Status = p.Status + res.Proposal = &p + return res +} + +// userFacingProposalError keeps known outcomes readable and hides storage +// details, which are logged instead. +func userFacingProposalError(cmd channelregistry.Command, err error) string { + var conflict errProposalConflict + switch { + case errors.As(err, &conflict), + errors.Is(err, errTooManyPending), errors.Is(err, errTooManyApproved), + errors.Is(err, errProposalNotFound), errors.Is(err, channelregistry.ErrInvalidCommand), + errors.Is(err, channelregistry.ErrNameEmpty), errors.Is(err, channelregistry.ErrNameTooLong), + errors.Is(err, channelregistry.ErrNameInvalidUTF8), errors.Is(err, channelregistry.ErrNameControl), + errors.Is(err, channelregistry.ErrNameSpace), errors.Is(err, channelregistry.ErrNameInvisible): + return err.Error() + } + log.Printf("[channel-proposals] %s %s failed: %v", cmd.Op, cmd.RequestID, err) + return errProposalStorage.Error() +} + +// Prune applies retention to the table and the queue's result files. +func (r *channelProposalRunner) Prune(ctx context.Context) { + now := r.now() + cutoff := now.Add(-time.Duration(r.limits.RetentionDays) * 24 * time.Hour).UnixMilli() + if n, err := r.store.PruneChannelProposals(ctx, cutoff); err != nil { + if !channelregistry.IsMissingTable(err) { + log.Printf("[channel-proposals] retention failed: %v", err) + } + } else if n > 0 { + log.Printf("[channel-proposals] retention removed %d old suggestion(s)", n) + } + if _, err := r.queue.Prune(now, r.limits.RequestTTL, r.limits.MaxResults); err != nil { + log.Printf("[channel-proposals] queue prune failed: %v", err) + } +} + +// Start runs RunOnce every tick and Prune every pruneEvery. The returned func +// stops the loop and waits for it. +func (r *channelProposalRunner) Start(tick, pruneEvery time.Duration) (stop func()) { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + t := time.NewTicker(tick) + defer t.Stop() + lastPrune := time.Time{} + for { + r.RunOnce(ctx) + if time.Since(lastPrune) >= pruneEvery { + r.Prune(ctx) + lastPrune = time.Now() + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } + }() + return func() { + cancel() + <-done + } +} diff --git a/cmd/ingestor/channel_proposals_test.go b/cmd/ingestor/channel_proposals_test.go new file mode 100644 index 000000000..429fbbf1a --- /dev/null +++ b/cmd/ingestor/channel_proposals_test.go @@ -0,0 +1,795 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/meshcore-analyzer/channelregistry" +) + +type proposalFixture struct { + t *testing.T + dbPath string + store *Store + keys *hotKeys + runner *channelProposalRunner + clock time.Time +} + +func newProposalFixture(t *testing.T, cfg *channelregistry.Config) *proposalFixture { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "meshcore.db") + store, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + f := &proposalFixture{t: t, dbPath: dbPath, store: store, clock: time.UnixMilli(1_700_000_000_000)} + f.keys = newHotKeys(map[string]string{"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72"}, nil) + f.runner = newChannelProposalRunner(store, f.keys, cfg) + f.runner.now = func() time.Time { return f.clock } + return f +} + +// enqueue writes a command the way the server does and returns its request id. +func (f *proposalFixture) enqueue(op, nameOrID string) string { + f.t.Helper() + cmd := channelregistry.Command{RequestID: channelregistry.NewID(), Op: op, CreatedAt: f.clock.UnixMilli()} + if op == channelregistry.OpSubmit { + cmd.Name = nameOrID + } else { + cmd.ProposalID = nameOrID + } + if err := f.runner.queue.Enqueue(cmd, 1000); err != nil { + f.t.Fatal(err) + } + return cmd.RequestID +} + +// run applies the queue and returns the status of request id. +func (f *proposalFixture) run(id string) channelregistry.RequestStatus { + f.t.Helper() + f.runner.RunOnce(context.Background()) + st, err := f.runner.queue.Lookup(id) + if err != nil { + f.t.Fatalf("lookup %s: %v", id, err) + } + return st +} + +func (f *proposalFixture) count(where string) int { + f.t.Helper() + var n int + if err := f.store.db.QueryRow(`SELECT COUNT(*) FROM channel_proposals WHERE ` + where).Scan(&n); err != nil { + f.t.Fatal(err) + } + return n +} + +func TestChannelProposalSubmitDuplicatesAndCase(t *testing.T) { + f := newProposalFixture(t, nil) + st := f.run(f.enqueue(channelregistry.OpSubmit, " Test ")) + if st.Status != channelregistry.RequestPending || st.Proposal == nil || st.Proposal.Name != "#Test" { + t.Fatalf("submit = %+v", st) + } + if st.Proposal.CreatedAt != f.clock.UnixMilli() || st.Proposal.ReviewedAt != nil { + t.Fatalf("timestamps = %+v", st.Proposal) + } + first := st.Proposal.ID + + // Same name from another request: reported, not duplicated. + dup := f.run(f.enqueue(channelregistry.OpSubmit, "#Test")) + if dup.Status != channelregistry.RequestPending || dup.Proposal == nil || dup.Proposal.ID != first { + t.Fatalf("duplicate = %+v", dup) + } + // Different case is a different channel (different key). + lower := f.run(f.enqueue(channelregistry.OpSubmit, "#test")) + if lower.Status != channelregistry.RequestPending || lower.Proposal.ID == first { + t.Fatalf("#test must be its own proposal: %+v", lower) + } + if n := f.count("1=1"); n != 2 { + t.Fatalf("rows = %d, want 2", n) + } + // Invalid names are rejected by the ingestor too (the queue is not trusted). + bad := f.run(f.enqueue(channelregistry.OpSubmit, "#"+strings.Repeat("x", 40))) + if bad.Status != channelregistry.RequestError || bad.Error != channelregistry.ErrNameTooLong.Error() { + t.Fatalf("too long = %+v", bad) + } +} + +func TestChannelProposalLimits(t *testing.T) { + f := newProposalFixture(t, &channelregistry.Config{MaxPending: 2, MaxApproved: 1}) + a := f.run(f.enqueue(channelregistry.OpSubmit, "#a")).Proposal.ID + b := f.run(f.enqueue(channelregistry.OpSubmit, "#b")).Proposal.ID + over := f.run(f.enqueue(channelregistry.OpSubmit, "#c")) + if over.Status != channelregistry.RequestError || over.Error != errTooManyPending.Error() { + t.Fatalf("pending limit = %+v", over) + } + if st := f.run(f.enqueue(channelregistry.OpApprove, a)); st.Status != channelregistry.RequestApproved { + t.Fatalf("approve a = %+v", st) + } + full := f.run(f.enqueue(channelregistry.OpApprove, b)) + if full.Status != channelregistry.RequestError || full.Error != errTooManyApproved.Error() { + t.Fatalf("approved limit = %+v", full) + } + if _, ok := f.keys.Channels()["#b"]; ok { + t.Fatal("#b must not be keyed when its approval failed") + } +} + +func TestChannelProposalReviewTransitions(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#test")).Proposal.ID + f.clock = f.clock.Add(time.Minute) + + st := f.run(f.enqueue(channelregistry.OpApprove, id)) + if st.Status != channelregistry.RequestApproved || st.Proposal.ReviewedAt == nil || *st.Proposal.ReviewedAt != f.clock.UnixMilli() { + t.Fatalf("approve = %+v", st) + } + // Firmware docs: hashtag channel "#test" has key 9cd8fcf22a47333b591d96a2b848b73f. + if got := f.keys.Channels()["#test"]; got != "9cd8fcf22a47333b591d96a2b848b73f" { + t.Fatalf("approved key = %q", got) + } + if f.keys.Channels()["Public"] == "" { + t.Fatal("approval dropped a configured key") + } + // Repeating the same decision is idempotent. + if again := f.run(f.enqueue(channelregistry.OpApprove, id)); again.Status != channelregistry.RequestApproved || again.Error != "" { + t.Fatalf("repeat approve = %+v", again) + } + // A conflicting decision fails and reports the stored state. + conflict := f.run(f.enqueue(channelregistry.OpReject, id)) + if conflict.Status != channelregistry.RequestError || conflict.Error != "suggestion was already approved" || + conflict.Proposal == nil || conflict.Proposal.Status != channelregistry.StatusApproved { + t.Fatalf("conflicting reject = %+v", conflict) + } + if missing := f.run(f.enqueue(channelregistry.OpApprove, "ffffffffffffffff")); missing.Error != errProposalNotFound.Error() { + t.Fatalf("unknown id = %+v", missing) + } + + other := f.run(f.enqueue(channelregistry.OpSubmit, "#other")).Proposal.ID + rej := f.run(f.enqueue(channelregistry.OpReject, other)) + if rej.Status != channelregistry.RequestRejected || rej.Proposal.ReviewedAt == nil { + t.Fatalf("reject = %+v", rej) + } + if _, ok := f.keys.Channels()["#other"]; ok { + t.Fatal("rejected channel must not be keyed") + } + if late := f.run(f.enqueue(channelregistry.OpApprove, other)); late.Error != "suggestion was already rejected" { + t.Fatalf("approve after reject = %+v", late) + } +} + +// A crash after the commit but before the result is written leaves the +// command in the queue. Applying it again must not duplicate or fail. +func TestChannelProposalCrashReplayIsIdempotent(t *testing.T) { + f := newProposalFixture(t, nil) + ctx := context.Background() + + submitID := f.enqueue(channelregistry.OpSubmit, "#Replay") + pending, _ := f.runner.queue.Pending() + f.runner.apply(ctx, pending[0].Command) // committed, "crash" before Complete + st := f.run(submitID) + if st.Status != channelregistry.RequestPending || st.Proposal.ID != submitID { + t.Fatalf("replayed submit = %+v", st) + } + if n := f.count("1=1"); n != 1 { + t.Fatalf("rows after replay = %d", n) + } + + approveID := f.enqueue(channelregistry.OpApprove, submitID) + pending, _ = f.runner.queue.Pending() + f.runner.apply(ctx, pending[0].Command) // approved in the DB, keys not updated yet + if _, ok := f.keys.Channels()["#Replay"]; ok { + t.Fatal("precondition: keys must not be updated by apply alone") + } + st = f.run(approveID) + if st.Status != channelregistry.RequestApproved || st.Error != "" { + t.Fatalf("replayed approve = %+v", st) + } + if _, ok := f.keys.Channels()["#Replay"]; !ok { + t.Fatal("replayed approval must add the key") + } +} + +func TestChannelProposalExpiredAndMalformedCommands(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.enqueue(channelregistry.OpSubmit, "#late") + f.clock = f.clock.Add(25 * time.Hour) + if st := f.run(id); st.Status != channelregistry.RequestError || st.Error != errRequestExpired.Error() { + t.Fatalf("expired = %+v", st) + } + if n := f.count("1=1"); n != 0 { + t.Fatalf("expired command wrote %d row(s)", n) + } + junk := filepath.Join(f.runner.queue.Dir(), "cmd-0123456789abcdef.json") + if err := os.WriteFile(junk, []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + f.runner.RunOnce(context.Background()) + if _, err := os.Stat(junk); !os.IsNotExist(err) { + t.Fatal("malformed command must be discarded") + } +} + +func TestChannelProposalRetention(t *testing.T) { + f := newProposalFixture(t, &channelregistry.Config{RetentionDays: 30}) + day := int64(24 * 60 * 60 * 1000) + now := f.clock.UnixMilli() + rows := []struct { + id, name, status string + created int64 + reviewed interface{} + }{ + {"1111111111111111", "#oldRejected", "rejected", now - 40*day, now - 31*day}, + {"2222222222222222", "#newRejected", "rejected", now - 40*day, now - 1*day}, + {"3333333333333333", "#oldPending", "pending", now - 31*day, nil}, + {"4444444444444444", "#newPending", "pending", now - 1*day, nil}, + {"5555555555555555", "#oldApproved", "approved", now - 400*day, now - 399*day}, + } + for _, r := range rows { + if _, err := f.store.db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at, reviewed_at) VALUES (?, ?, ?, ?, ?)`, + r.id, r.name, r.status, r.created, r.reviewed); err != nil { + t.Fatal(err) + } + } + f.runner.Prune(context.Background()) + var names []string + got, err := channelregistry.ListProposals(context.Background(), f.store.db, "", 0) + if err != nil { + t.Fatal(err) + } + for _, p := range got { + names = append(names, p.Name) + } + // Newest first; the old rejected and old pending rows are gone, the + // approved one is kept however old it is. + if got := strings.Join(names, ","); got != "#newPending,#newRejected,#oldApproved" { + t.Fatalf("kept = %s", got) + } +} + +// Approvals survive a restart: a fresh process loads them into the keys. +func TestChannelProposalRestartPersistence(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#Persist")).Proposal.ID + pendingID := f.run(f.enqueue(channelregistry.OpSubmit, "#StillPending")).Proposal.ID + f.run(f.enqueue(channelregistry.OpApprove, id)) + f.store.Close() + + store, err := OpenStore(f.dbPath) + if err != nil { + t.Fatal(err) + } + defer store.Close() + keys := newHotKeys(map[string]string{}, nil) + runner := newChannelProposalRunner(store, keys, nil) + n, err := runner.LoadApproved(context.Background()) + if err != nil || n != 1 { + t.Fatalf("LoadApproved = %d, %v", n, err) + } + if keys.Channels()["#Persist"] != deriveHashtagChannelKey("#Persist") { + t.Fatal("approved channel not keyed after restart") + } + p, found, err := channelregistry.GetProposal(context.Background(), store.db, pendingID) + if err != nil || !found || p.Status != channelregistry.StatusPending { + t.Fatalf("pending proposal after restart = %+v, %v, %v", p, found, err) + } +} + +// SIGHUP rebuilds the config-derived keys; approved channels must survive +// it, and a manually configured key for the same name keeps priority. +func TestChannelProposalKeysSurviveSIGHUPAndConfigWins(t *testing.T) { + configPath := writeTestConfig(t, []string{"#alpha"}, nil) + cfg, err := LoadConfig(configPath) + if err != nil { + t.Fatal(err) + } + hk := newHotKeys(loadChannelKeys(cfg, configPath), loadRegionKeys(cfg)) + if hk.AddApproved("#Shared", "#Manual") != 2 || hk.AddApproved("#Shared") != 0 { + t.Fatal("AddApproved must report only new names") + } + + manualKey := "00112233445566778899aabbccddeeff" + if err := os.WriteFile(configPath, mustJSON(t, struct { + HashChannels []string `json:"hashChannels"` + ChannelKeys map[string]string `json:"channelKeys"` + }{HashChannels: []string{"#alpha", "#beta"}, ChannelKeys: map[string]string{"#Manual": manualKey}}), 0o644); err != nil { + t.Fatal(err) + } + stop := startSIGHUPReload(hk, configPath) + defer stop() + if err := syscall.Kill(os.Getpid(), syscall.SIGHUP); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if _, ok := hk.Channels()["#beta"]; ok { + break + } + time.Sleep(10 * time.Millisecond) + } + ch := hk.Channels() + if _, ok := ch["#beta"]; !ok { + t.Fatalf("SIGHUP did not reload: %v", ch) + } + if ch["#Shared"] != deriveHashtagChannelKey("#Shared") { + t.Fatal("SIGHUP dropped an approved channel") + } + if ch["#Manual"] != manualKey { + t.Fatalf("configured key must win over the approved one, got %q", ch["#Manual"]) + } +} + +// The merged snapshot handed to readers is never mutated afterwards. +func TestHotKeysSnapshotIsImmutable(t *testing.T) { + hk := newHotKeys(map[string]string{"#a": "1"}, nil) + snap := hk.Channels() + hk.AddApproved("#b") + if _, ok := snap["#b"]; ok { + t.Fatal("AddApproved mutated a published snapshot") + } + if _, ok := hk.Channels()["#b"]; !ok { + t.Fatal("new snapshot lacks #b") + } +} + +// A database without the table (a server-side rolling-upgrade state) reads +// as empty rather than failing. +func TestChannelProposalMissingTableReadsEmpty(t *testing.T) { + f := newProposalFixture(t, nil) + if _, err := f.store.db.Exec(`DROP TABLE channel_proposals`); err != nil { + t.Fatal(err) + } + if n, err := f.runner.LoadApproved(context.Background()); err != nil || n != 0 { + t.Fatalf("LoadApproved on missing table = %d, %v", n, err) + } + list, err := channelregistry.ListProposals(context.Background(), f.store.db, "", 0) + if err != nil || len(list) != 0 { + t.Fatalf("ListProposals on missing table = %v, %v", list, err) + } + if _, found, err := channelregistry.GetProposal(context.Background(), f.store.db, "0123456789abcdef"); err != nil || found { + t.Fatalf("GetProposal on missing table = %v, %v", found, err) + } + f.runner.Prune(context.Background()) // must not panic or log-spam +} + +// ── revoke ────────────────────────────────────────────────────────────── + +// Approve, then revoke: the row moves to revoked, reviewed_at is updated +// again, and the key drops out of the merged map handleMessage actually +// reads for decoding (main.go: handleMessage(..., keys.Channels(), ...)) — +// unless a manually configured base key for the same name exists, in which +// case decoding must keep working. +func TestChannelProposalRevokeRemovesKeyUnlessBaseConfigured(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#Revocable")).Proposal.ID + f.run(f.enqueue(channelregistry.OpApprove, id)) + if _, ok := f.keys.Channels()["#Revocable"]; !ok { + t.Fatal("precondition: approved channel must be keyed") + } + f.clock = f.clock.Add(time.Minute) + + st := f.run(f.enqueue(channelregistry.OpRevoke, id)) + if st.Status != channelregistry.RequestRevoked || st.Proposal == nil || + st.Proposal.ReviewedAt == nil || *st.Proposal.ReviewedAt != f.clock.UnixMilli() { + t.Fatalf("revoke = %+v", st) + } + if _, ok := f.keys.Channels()["#Revocable"]; ok { + t.Fatal("revoked channel must no longer be keyed — traffic on it must not decrypt") + } + p, found, err := channelregistry.GetProposal(context.Background(), f.store.db, id) + if err != nil || !found || p.Status != channelregistry.StatusRevoked { + t.Fatalf("row after revoke = %+v, %v, %v", p, found, err) + } + + // Now with a manually configured base key for the same name: base must + // win, so the channel keeps decrypting even after the approval that + // added it via `approved` is revoked. + f2 := newProposalFixture(t, nil) + f2.keys.setBase(map[string]string{"#Revocable": "deadbeefdeadbeefdeadbeefdeadbeef"}) + id2 := f2.run(f2.enqueue(channelregistry.OpSubmit, "#Revocable")).Proposal.ID + f2.run(f2.enqueue(channelregistry.OpApprove, id2)) + f2.run(f2.enqueue(channelregistry.OpRevoke, id2)) + if got := f2.keys.Channels()["#Revocable"]; got != "deadbeefdeadbeefdeadbeefdeadbeef" { + t.Fatalf("base key must survive revoking the approval for the same name, got %q", got) + } +} + +// Replaying the exact same revoke command twice is a no-op success both +// times: no error, no further key-map mutation, no row change on the second +// application. +func TestChannelProposalRevokeIsIdempotent(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#Idem")).Proposal.ID + f.run(f.enqueue(channelregistry.OpApprove, id)) + + reqID := f.enqueue(channelregistry.OpRevoke, id) + first := f.run(reqID) + if first.Status != channelregistry.RequestRevoked || first.Error != "" { + t.Fatalf("first revoke = %+v", first) + } + // Re-run the queue again (nothing pending, harmless) then replay a fresh + // revoke command for the same, already-revoked proposal. + second := f.run(f.enqueue(channelregistry.OpRevoke, id)) + if second.Status != channelregistry.RequestRevoked || second.Error != "" { + t.Fatalf("second revoke (already revoked) = %+v", second) + } + if n := f.count(`id = '` + id + `' AND status = 'revoked'`); n != 1 { + t.Fatalf("row count after double revoke = %d", n) + } +} + +// A crash after the commit but before the result is written must not double +// -apply or corrupt state when the revoke command is replayed. +func TestChannelProposalRevokeCrashReplayIsIdempotent(t *testing.T) { + f := newProposalFixture(t, nil) + ctx := context.Background() + id := f.run(f.enqueue(channelregistry.OpSubmit, "#CrashRevoke")).Proposal.ID + f.run(f.enqueue(channelregistry.OpApprove, id)) + + revokeID := f.enqueue(channelregistry.OpRevoke, id) + pending, _ := f.runner.queue.Pending() + f.runner.apply(ctx, pending[0].Command) // committed, "crash" before Complete + if _, ok := f.keys.Channels()["#CrashRevoke"]; !ok { + t.Fatal("precondition: apply alone must not touch keys yet") + } + st := f.run(revokeID) // replays the same command via RunOnce + if st.Status != channelregistry.RequestRevoked || st.Error != "" { + t.Fatalf("replayed revoke = %+v", st) + } + if _, ok := f.keys.Channels()["#CrashRevoke"]; ok { + t.Fatal("replayed revoke must still remove the key") + } + if n := f.count(`id = '` + id + `' AND status = 'revoked'`); n != 1 { + t.Fatalf("rows after crash-replay = %d", n) + } +} + +// approve → revoke → approve again: the second approve is a fresh decision +// on a row that is no longer pending (it is revoked), so it must be a +// conflict, not a silent re-approval. +func TestChannelProposalApproveRevokeApproveIsConflictNotSilentReapproval(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#Cycle")).Proposal.ID + f.run(f.enqueue(channelregistry.OpApprove, id)) + f.run(f.enqueue(channelregistry.OpRevoke, id)) + + again := f.run(f.enqueue(channelregistry.OpApprove, id)) + if again.Status != channelregistry.RequestError || again.Error != "suggestion was already revoked" { + t.Fatalf("approve of a revoked row = %+v, want conflict", again) + } + p, _, err := channelregistry.GetProposal(context.Background(), f.store.db, id) + if err != nil || p.Status != channelregistry.StatusRevoked { + t.Fatalf("status must stay revoked after the rejected re-approval attempt: %+v, %v", p, err) + } + if _, ok := f.keys.Channels()["#Cycle"]; ok { + t.Fatal("the conflicting approve attempt must not have re-added the key") + } +} + +// Revoking a pending, a rejected, or an unknown id are all conflicts/not- +// found: no DB mutation beyond what already existed, no key-map mutation. +func TestChannelProposalRevokeOfNonApprovedProducesConflict(t *testing.T) { + f := newProposalFixture(t, nil) + pendingID := f.run(f.enqueue(channelregistry.OpSubmit, "#StillPending")).Proposal.ID + rejectedID := f.run(f.enqueue(channelregistry.OpSubmit, "#WillReject")).Proposal.ID + f.run(f.enqueue(channelregistry.OpReject, rejectedID)) + + if st := f.run(f.enqueue(channelregistry.OpRevoke, pendingID)); st.Status != channelregistry.RequestError || st.Error != "suggestion is not approved (it is pending)" { + t.Fatalf("revoke pending = %+v", st) + } + if st := f.run(f.enqueue(channelregistry.OpRevoke, rejectedID)); st.Status != channelregistry.RequestError || st.Error != "suggestion is not approved (it is rejected)" { + t.Fatalf("revoke rejected = %+v", st) + } + if st := f.run(f.enqueue(channelregistry.OpRevoke, "ffffffffffffffff")); st.Status != channelregistry.RequestError || st.Error != errProposalNotFound.Error() { + t.Fatalf("revoke unknown = %+v", st) + } + p, _, _ := channelregistry.GetProposal(context.Background(), f.store.db, pendingID) + if p.Status != channelregistry.StatusPending { + t.Fatalf("pending row mutated by a failed revoke: %+v", p) + } + r, _, _ := channelregistry.GetProposal(context.Background(), f.store.db, rejectedID) + if r.Status != channelregistry.StatusRejected { + t.Fatalf("rejected row mutated by a failed revoke: %+v", r) + } + if _, ok := f.keys.Channels()["#StillPending"]; ok { + t.Fatal("a failed revoke must never add a key") + } +} + +// A revoke must not be resurrected by a reload (the SIGHUP handler's +// config-reread path — exercised here by calling hk.reload directly, which +// is exactly what startSIGHUPReload calls, so this is not a weaker test) +// unless the operator actually configured the channel; when they did, base +// wins (same rule as for a still-approved channel, already covered by +// TestChannelProposalKeysSurviveSIGHUPAndConfigWins). +func TestChannelProposalRevokedKeySurvivesReloadUnlessBaseConfigured(t *testing.T) { + configPath := writeTestConfig(t, nil, nil) + cfg, err := LoadConfig(configPath) + if err != nil { + t.Fatal(err) + } + hk := newHotKeys(loadChannelKeys(cfg, configPath), loadRegionKeys(cfg)) + hk.AddApproved("#Revocable", "#StillApproved") + if !hk.RemoveApproved("#Revocable") { + t.Fatal("RemoveApproved must report the name was present") + } + if hk.RemoveApproved("#Revocable") { + t.Fatal("RemoveApproved must report false the second time") + } + if _, ok := hk.Channels()["#Revocable"]; ok { + t.Fatal("precondition: must be unkeyed right after RemoveApproved") + } + + // A config reload that does not mention the channel at all: it must stay + // absent, not come back from anywhere, and an unrelated still-approved + // channel must survive the reload untouched (reload must never wipe or + // otherwise mutate the approved layer, only base). + if err := hk.reload(configPath); err != nil { + t.Fatal(err) + } + if _, ok := hk.Channels()["#Revocable"]; ok { + t.Fatal("reload must not resurrect a revoked channel that is not configured") + } + if _, ok := hk.Channels()["#StillApproved"]; !ok { + t.Fatal("reload must not wipe unrelated approved channels") + } + + // Now #Revocable is configured under hashChannels: base wins, same as + // for any other manually configured key. + if err := os.WriteFile(configPath, mustJSON(t, struct { + HashChannels []string `json:"hashChannels"` + }{HashChannels: []string{"#Revocable"}}), 0o644); err != nil { + t.Fatal(err) + } + if err := hk.reload(configPath); err != nil { + t.Fatal(err) + } + if _, ok := hk.Channels()["#Revocable"]; !ok { + t.Fatal("a channel later configured under hashChannels must decrypt again, even though it was revoked") + } +} + +// Process restart after a revoke: LoadApproved only ever selects +// status = 'approved' (internal/channelregistry/sql.go), so a fresh process +// never re-adds a revoked channel to the keys. +func TestChannelProposalRevokeSurvivesRestart(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#RestartRevoke")).Proposal.ID + f.run(f.enqueue(channelregistry.OpApprove, id)) + f.run(f.enqueue(channelregistry.OpRevoke, id)) + f.store.Close() + + store, err := OpenStore(f.dbPath) + if err != nil { + t.Fatal(err) + } + defer store.Close() + keys := newHotKeys(map[string]string{}, nil) + runner := newChannelProposalRunner(store, keys, nil) + n, err := runner.LoadApproved(context.Background()) + if err != nil || n != 0 { + t.Fatalf("LoadApproved after restart = %d, %v — a revoked channel must not be reloaded", n, err) + } + if _, ok := keys.Channels()["#RestartRevoke"]; ok { + t.Fatal("revoked channel reappeared after restart") + } +} + +// Revoke against a database without the channel_proposals table behaves +// like the other missing-table paths: no panic, a sane not-found error. +func TestChannelProposalRevokeMissingTable(t *testing.T) { + f := newProposalFixture(t, nil) + if _, err := f.store.db.Exec(`DROP TABLE channel_proposals`); err != nil { + t.Fatal(err) + } + st := f.run(f.enqueue(channelregistry.OpRevoke, "0123456789abcdef")) + if st.Status != channelregistry.RequestError { + t.Fatalf("revoke on missing table = %+v", st) + } +} + +// Re-suggesting the same name after a revoke lands as pending again (never +// auto-approved), reuses the SAME row id (UNIQUE(name)), and a replay of +// that exact resubmission request is itself idempotent. +func TestChannelProposalResubmitAfterRevokeLandsPendingWithSameID(t *testing.T) { + f := newProposalFixture(t, nil) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#Resuggest")).Proposal.ID + f.run(f.enqueue(channelregistry.OpApprove, id)) + f.run(f.enqueue(channelregistry.OpRevoke, id)) + if _, ok := f.keys.Channels()["#Resuggest"]; ok { + t.Fatal("precondition: must be revoked (unkeyed) before resuggesting") + } + + resubmitReqID := f.enqueue(channelregistry.OpSubmit, "#Resuggest") + st := f.run(resubmitReqID) + if st.Status != channelregistry.RequestPending { + t.Fatalf("resubmit after revoke = %+v, want pending (never auto-approved)", st) + } + if st.Proposal.ID != id { + t.Fatalf("resubmit after revoke got a new id %q, want the original %q", st.Proposal.ID, id) + } + if n := f.count("1=1"); n != 1 { + t.Fatalf("resubmit after revoke must reuse the row, got %d rows", n) + } + if _, ok := f.keys.Channels()["#Resuggest"]; ok { + t.Fatal("resubmission must not re-add the key: it is pending, not approved") + } + + // Replay of the exact same resubmission request (e.g. after a crash + // before its result was written) must be idempotent: same id, no new row. + pending, err := channelregistry.ListProposals(context.Background(), f.store.db, "", 0) + if err != nil || len(pending) != 1 { + t.Fatalf("listing before replay: %v, %v", pending, err) + } + replayCmd := channelregistry.Command{RequestID: resubmitReqID, Op: channelregistry.OpSubmit, Name: "#Resuggest", CreatedAt: f.clock.UnixMilli()} + replayed, err := f.store.submitChannelProposal(context.Background(), replayCmd, 100, f.clock.UnixMilli()) + if err != nil { + t.Fatalf("replayed resubmit: %v", err) + } + if replayed.ID != id || replayed.Status != channelregistry.StatusPending { + t.Fatalf("replayed resubmit = %+v, want id=%s status=pending", replayed, id) + } + if n := f.count("1=1"); n != 1 { + t.Fatalf("rows after replayed resubmit = %d, want 1 (still idempotent)", n) + } +} + +// Retention: a revoked row older than the cutoff is pruned exactly like a +// rejected one; a newer one is kept; an approved row is never pruned. +func TestChannelProposalRevokeRetention(t *testing.T) { + f := newProposalFixture(t, &channelregistry.Config{RetentionDays: 30}) + day := int64(24 * 60 * 60 * 1000) + now := f.clock.UnixMilli() + rows := []struct { + id, name, status string + created int64 + reviewed interface{} + }{ + {"6666666666666666", "#oldRevoked", "revoked", now - 400*day, now - 31*day}, + {"7777777777777777", "#newRevoked", "revoked", now - 400*day, now - 1*day}, + {"8888888888888888", "#stillApproved", "approved", now - 400*day, now - 399*day}, + } + for _, r := range rows { + if _, err := f.store.db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at, reviewed_at) VALUES (?, ?, ?, ?, ?)`, + r.id, r.name, r.status, r.created, r.reviewed); err != nil { + t.Fatal(err) + } + } + f.runner.Prune(context.Background()) + var names []string + got, err := channelregistry.ListProposals(context.Background(), f.store.db, "", 0) + if err != nil { + t.Fatal(err) + } + for _, p := range got { + names = append(names, p.Name) + } + if got := strings.Join(names, ","); got != "#newRevoked,#stillApproved" { + t.Fatalf("kept = %s, want #newRevoked,#stillApproved", got) + } +} + +// A rejected name stays blocked: re-suggesting it reports the earlier +// rejection instead of opening a new pending suggestion, until retention +// (retentionDays, 30 by default) removes the rejected row. Only a revoked +// name may be suggested again straight away. +func TestChannelProposalRejectedNameBlockedUntilRetention(t *testing.T) { + f := newProposalFixture(t, &channelregistry.Config{RetentionDays: 30}) + id := f.run(f.enqueue(channelregistry.OpSubmit, "#Spam")).Proposal.ID + if st := f.run(f.enqueue(channelregistry.OpReject, id)); st.Status != channelregistry.RequestRejected { + t.Fatalf("reject = %+v", st) + } + again := f.run(f.enqueue(channelregistry.OpSubmit, "#Spam")) + if again.Status != channelregistry.RequestRejected || again.Proposal == nil || again.Proposal.ID != id { + t.Fatalf("re-suggesting a rejected name = %+v, want the earlier rejection", again) + } + if n := f.count("status = 'pending'"); n != 0 { + t.Fatalf("re-suggestion opened %d pending row(s)", n) + } + if n := f.count("1=1"); n != 1 { + t.Fatalf("rows = %d, want 1", n) + } + + // 29 days later it is still blocked; after 31 days retention frees it. + f.clock = f.clock.Add(29 * 24 * time.Hour) + f.runner.Prune(context.Background()) + if st := f.run(f.enqueue(channelregistry.OpSubmit, "#Spam")); st.Status != channelregistry.RequestRejected { + t.Fatalf("day 29 re-suggestion = %+v, want still rejected", st) + } + f.clock = f.clock.Add(2 * 24 * time.Hour) + f.runner.Prune(context.Background()) + fresh := f.run(f.enqueue(channelregistry.OpSubmit, "#Spam")) + if fresh.Status != channelregistry.RequestPending || fresh.Proposal == nil || fresh.Proposal.ID == id { + t.Fatalf("after retention re-suggestion = %+v, want a new pending suggestion", fresh) + } +} + +// The ingestor never trusts the queue file: an invisible formatting +// character is rejected here too, with the user-facing message. +func TestChannelProposalIngestorRejectsInvisibleCharacters(t *testing.T) { + f := newProposalFixture(t, nil) + for _, name := range []string{"#a\u200bb", "#\ufeffab", "#ab\u00adc", "#a\U000e0061b", "#a\u2028b"} { + st := f.run(f.enqueue(channelregistry.OpSubmit, name)) + if st.Status != channelregistry.RequestError || + (st.Error != channelregistry.ErrNameInvisible.Error() && st.Error != channelregistry.ErrNameControl.Error()) { + t.Errorf("submit %q = %+v, want the name error", name, st) + } + } + if n := f.count("1=1"); n != 0 { + t.Fatalf("rows = %d, want 0", n) + } + ok := f.run(f.enqueue(channelregistry.OpSubmit, "#\U0001f3f3\ufe0f\u200d\U0001f308")) + if ok.Status != channelregistry.RequestPending { + t.Fatalf("emoji with ZWJ and VS16 = %+v", ok) + } +} + +// The ingestor publishes the hashtag names its config-derived layer already +// decrypts, at startup and again after every reload, so the server can mark +// them as built in. It rewrites the file only when that layer changed. +func TestChannelProposalPublishesBuiltinNames(t *testing.T) { + f := newProposalFixture(t, nil) + f.keys.setBase(map[string]string{"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72", "#test": deriveHashtagChannelKey("#test")}) + f.keys.AddApproved("#Approved") + f.runner.RunOnce(context.Background()) + names, err := f.runner.queue.ReadBuiltinNames() + if err != nil { + t.Fatal(err) + } + if len(names) != 1 || !names["#test"] { + t.Fatalf("builtin names = %v, want only #test (approved names are not built in)", names) + } + + // Unchanged base: no rewrite on later ticks. + if err := os.Remove(f.runner.queue.BuiltinNamesPath()); err != nil { + t.Fatal(err) + } + f.runner.RunOnce(context.Background()) + if _, err := os.Stat(f.runner.queue.BuiltinNamesPath()); !os.IsNotExist(err) { + t.Fatalf("file rewritten although the base layer did not change: %v", err) + } + + // A reload (setBase) publishes the new set on the next tick. + f.keys.setBase(map[string]string{"#chat": deriveHashtagChannelKey("#chat"), "#Approved": "00112233445566778899aabbccddeeff"}) + f.runner.RunOnce(context.Background()) + names, err = f.runner.queue.ReadBuiltinNames() + if err != nil { + t.Fatal(err) + } + if len(names) != 2 || !names["#chat"] || !names["#Approved"] { + t.Fatalf("after reload builtin names = %v", names) + } +} + +// The real rainbow table ships common names; they are published as built in. +func TestChannelProposalBuiltinNamesIncludeRainbowTable(t *testing.T) { + rainbow := filepath.Join("..", "..", "channel-rainbow.json") + if _, err := os.Stat(rainbow); err != nil { + t.Skipf("rainbow table not found: %v", err) + } + t.Setenv("CHANNEL_KEYS_PATH", rainbow) + configPath := writeTestConfig(t, nil, nil) + cfg, err := LoadConfig(configPath) + if err != nil { + t.Fatal(err) + } + f := newProposalFixture(t, nil) + f.keys.setBase(loadChannelKeys(cfg, configPath)) + f.runner.RunOnce(context.Background()) + names, err := f.runner.queue.ReadBuiltinNames() + if err != nil { + t.Fatal(err) + } + for _, n := range []string{"#test", "#chat", "#general"} { + if !names[n] { + t.Errorf("%s from channel-rainbow.json must be published as built in (%d names)", n, len(names)) + } + } +} diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index be1d55f4f..9b0c08581 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/meshcore-analyzer/channelregistry" "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/geofilter" ) @@ -88,6 +89,11 @@ type Config struct { // (#1608). Received messages are drained once the write path is ready. // 0 / unset => default. Bounded memory. IngestBufferSize int `json:"ingestBufferSize,omitempty"` + + // ChannelProposals: limits for publicly suggested hashtag channels + // (internal/channelregistry). Approved channels are always loaded into + // the channel keys, whether or not new submissions are enabled. + ChannelProposals *channelregistry.Config `json:"channelProposals,omitempty"` } // NeighborEdgesDaysOrDefault returns the configured pruning window or 5. diff --git a/cmd/ingestor/go.mod b/cmd/ingestor/go.mod index 3d440215c..988ba3b61 100644 --- a/cmd/ingestor/go.mod +++ b/cmd/ingestor/go.mod @@ -55,3 +55,7 @@ replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue require github.com/meshcore-analyzer/regions v0.0.0 replace github.com/meshcore-analyzer/regions => ../../internal/regions + +require github.com/meshcore-analyzer/channelregistry v0.0.0 + +replace github.com/meshcore-analyzer/channelregistry => ../../internal/channelregistry diff --git a/cmd/ingestor/hot_reload.go b/cmd/ingestor/hot_reload.go index 624017603..46aba1d10 100644 --- a/cmd/ingestor/hot_reload.go +++ b/cmd/ingestor/hot_reload.go @@ -4,6 +4,7 @@ import ( "log" "os" "os/signal" + "sync" "sync/atomic" "syscall" ) @@ -14,19 +15,109 @@ import ( // loadChannelKeys nor loadRegionKeys re-runs automatically otherwise — // hashChannels/hashRegions additions in config.json require a full ingestor // restart to take effect without this. +// +// Channel keys come in two layers. base is what loadChannelKeys derives from +// config (builtin, rainbow, hashChannels, channelKeys) and is replaced on +// every SIGHUP. approved holds the publicly suggested hashtag channels an +// administrator approved (internal/channelregistry): approvals add to it and +// revocations remove from it (RemoveApproved), it is reloaded from the +// database at startup, and a SIGHUP never touches it, so a reload can neither +// drop an approved channel nor bring back a revoked one. channelKeys is the +// merged snapshot the MQTT handlers read, with base winning over approved, so +// a manually configured key for the same name keeps priority. It also means +// a name already in base (for example one of the ~320 rainbow-table names +// such as #test or #chat) gains nothing from approval and keeps decrypting +// after a revoke; BaseNames lets the proposal runner tell the server so. type hotKeys struct { channelKeys atomic.Pointer[map[string]string] regionKeys atomic.Pointer[map[string][]byte] + + mu sync.Mutex // serializes writers of base/approved and the merge + base map[string]string + approved map[string]string + baseGen uint64 // incremented by every setBase } // newHotKeys wraps the initial startup-loaded key maps. func newHotKeys(channelKeys map[string]string, regionKeys map[string][]byte) *hotKeys { - hk := &hotKeys{} - hk.channelKeys.Store(&channelKeys) + hk := &hotKeys{approved: make(map[string]string)} hk.regionKeys.Store(®ionKeys) + hk.setBase(channelKeys) return hk } +// setBase replaces the config-derived channel keys and republishes the merge. +func (hk *hotKeys) setBase(channelKeys map[string]string) { + hk.mu.Lock() + defer hk.mu.Unlock() + hk.base = channelKeys + hk.baseGen++ + hk.publishLocked() +} + +// BaseNames returns the names in the config-derived layer and its generation, +// which changes on every setBase (startup and each reload). +func (hk *hotKeys) BaseNames() ([]string, uint64) { + hk.mu.Lock() + defer hk.mu.Unlock() + names := make([]string, 0, len(hk.base)) + for name := range hk.base { + names = append(names, name) + } + return names, hk.baseGen +} + +// AddApproved adds approved hashtag channels, deriving each key with the same +// algorithm as hashChannels. It returns how many names were new. Names +// already approved are ignored, so repeated calls (startup load, replayed +// approvals) are harmless. +func (hk *hotKeys) AddApproved(names ...string) int { + hk.mu.Lock() + defer hk.mu.Unlock() + added := 0 + for _, name := range names { + if name == "" { + continue + } + if _, ok := hk.approved[name]; ok { + continue + } + hk.approved[name] = deriveHashtagChannelKey(name) + added++ + } + if added > 0 { + hk.publishLocked() + } + return added +} + +// RemoveApproved removes a name from the approved layer only (never touches +// base, so a manually configured key for the same channel keeps decrypting +// it). Returns whether it was present. +func (hk *hotKeys) RemoveApproved(name string) bool { + hk.mu.Lock() + defer hk.mu.Unlock() + if _, ok := hk.approved[name]; !ok { + return false + } + delete(hk.approved, name) + hk.publishLocked() + return true +} + +// publishLocked builds a fresh merged map (never mutating one a reader may +// hold) and swaps it in. O(base + approved), and only on reload/approval. +func (hk *hotKeys) publishLocked() { + merged := make(map[string]string, len(hk.base)+len(hk.approved)) + for name, key := range hk.approved { + merged[name] = key + } + for name, key := range hk.base { + merged[name] = key + } + hk.channelKeys.Store(&merged) +} + // Channels returns the current channel-decryption key snapshot. Safe to // call from any goroutine; cheap (one atomic load, no lock). func (hk *hotKeys) Channels() map[string]string { @@ -50,9 +141,9 @@ func (hk *hotKeys) reload(configPath string) error { } ck := loadChannelKeys(cfg, configPath) rk := loadRegionKeys(cfg) - hk.channelKeys.Store(&ck) + hk.setBase(ck) hk.regionKeys.Store(&rk) - log.Printf("[hot-reload] reloaded %d channel key(s), %d region key(s) from %s", len(ck), len(rk), configPath) + log.Printf("[hot-reload] reloaded %d channel key(s), %d region key(s) from %s (approved shared channels kept)", len(ck), len(rk), configPath) return nil } diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index d4513e575..3ab0626b1 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -113,6 +113,18 @@ func main() { stopSIGHUPReload := startSIGHUPReload(keys, *configPath) defer stopSIGHUPReload() + // Shared channel proposals: approved hashtag channels join the live keys + // before any MQTT traffic is handled, and the ingestor applies the + // server's queued suggestions/decisions (the server is read-only). + proposals := newChannelProposalRunner(store, keys, cfg.ChannelProposals) + if n, err := proposals.LoadApproved(context.Background()); err != nil { + log.Printf("[channel-proposals] loading approved channels failed: %v", err) + } else if n > 0 { + log.Printf("[channel-proposals] %d approved shared channel(s) added to channel keys", n) + } + stopProposals := proposals.Start(2*time.Second, time.Hour) + defer stopProposals() + // Subscribe-early + buffer (#1608): the MQTT subscription is brought up // before startup maintenance so no packets are missed while the single // SQLite writer is blocked (e.g. a large CREATE INDEX migration). Received diff --git a/cmd/server/channel_proposals.go b/cmd/server/channel_proposals.go new file mode 100644 index 000000000..bd4bf7524 --- /dev/null +++ b/cmd/server/channel_proposals.go @@ -0,0 +1,490 @@ +package main + +// Shared channel proposals — the read-only server side. +// +// Anyone may suggest a public hashtag channel; an administrator (the existing +// apiKey) approves or rejects it. The server never writes SQLite: every +// suggestion and decision becomes one atomic command file in the +// channelregistry queue next to the database, and the ingestor applies it and +// writes a result file the server reads back (cmd/ingestor/channel_proposals.go). +// +// Bounds: the queue holds at most MaxQueuedRequests commands, submissions are +// capped by a global sliding window of SubmissionsPerHour (a per-client limit +// would be unreliable behind a reverse proxy — see ws_limits.go), the ingestor +// enforces MaxPending/MaxApproved, and every list read is capped. + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math" + "net/http" + "os" + "sync" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/channelregistry" +) + +// approvedCacheTTL bounds how stale GET /api/channels' approvedChannels can +// be. A request-status read that observes an approval or revocation the +// cached snapshot may predate invalidates it early (see noteDecision). +const approvedCacheTTL = 10 * time.Second + +// maxProposalBodyBytes caps the POST body; a name is at most 31 bytes. +const maxProposalBodyBytes = 1024 + +// ChannelProposalConfigResponse is GET /api/channel-proposals/config. +type ChannelProposalConfigResponse struct { + Enabled bool `json:"enabled"` +} + +// ChannelProposalSubmitRequest is the body of POST /api/channel-proposals. +type ChannelProposalSubmitRequest struct { + Name string `json:"name"` +} + +// ChannelProposalAcceptedResponse is the 202 body of every queued request. +type ChannelProposalAcceptedResponse struct { + RequestID string `json:"requestId"` +} + +// AdminChannelProposal is one row of GET /api/admin/channel-proposals. +type AdminChannelProposal struct { + channelregistry.Proposal + // BuiltIn: the ingestor already decrypts this exact name through its + // config-derived keys (built-in, rainbow table, hashChannels, + // channelKeys), so approving it adds nothing and revoking it does not + // stop decryption. + BuiltIn bool `json:"builtIn,omitempty"` +} + +// ChannelProposalRequestResponse is GET /api/channel-proposals/requests/{id}. +type ChannelProposalRequestResponse struct { + channelregistry.RequestStatus + // BuiltIn: see AdminChannelProposal.BuiltIn; set for the proposal's name. + BuiltIn bool `json:"builtIn,omitempty"` +} + +// ChannelProposalListResponse is GET /api/admin/channel-proposals. +type ChannelProposalListResponse struct { + Proposals []AdminChannelProposal `json:"proposals"` + // Enabled reports whether public submissions are currently open. + Enabled bool `json:"enabled"` +} + +type approvedSnapshot struct { + channels []ApprovedChannel + builtAt time.Time // when the list was last read successfully + expires time.Time +} + +// builtinNamesCache is the last parsed builtin names file, keyed by its +// modification time and size so an unchanged file is not parsed again. +type builtinNamesCache struct { + mod time.Time + size int64 + names map[string]bool +} + +type channelProposalService struct { + enabled bool // public submissions open + limits channelregistry.Limits + queue *channelregistry.Queue // nil when there is no database path + db *sql.DB // read-only; nil without a database + now func() time.Time + + rateMu sync.Mutex + recent []time.Time // accepted submissions in the last hour, oldest first + + approvedMu sync.Mutex + approved *approvedSnapshot + + builtinMu sync.Mutex + builtin *builtinNamesCache +} + +func newChannelProposalService(cfg *Config, db *sql.DB, queueDir string) *channelProposalService { + var pc *channelregistry.Config + strongKey := false + if cfg != nil { + pc = cfg.ChannelProposals + strongKey = cfg.APIKey != "" && !IsWeakAPIKey(cfg.APIKey) + } + svc := &channelProposalService{ + limits: pc.Limits(), + db: db, + now: time.Now, + } + if queueDir != "" { + svc.queue = channelregistry.NewQueue(queueDir) + } + svc.enabled = pc.SubmissionsRequested() && strongKey && svc.queue != nil && db != nil + if pc.SubmissionsRequested() && !svc.enabled { + log.Printf("[channel-proposals] WARNING: channelProposals.enabled is set but public suggestions stay closed: " + + "they need a strong apiKey (16+ characters, not a placeholder) and a database") + } + return svc +} + +// channelProposals returns the service, building it on first use. +func (s *Server) channelProposals() *channelProposalService { + s.proposalsOnce.Do(func() { + if s.proposals != nil { + return + } + var conn *sql.DB + queueDir := "" + if s.db != nil { + conn = s.db.conn + if s.db.path != "" { + queueDir = channelregistry.QueueDir(s.db.path) + } + } + s.proposals = newChannelProposalService(s.cfg, conn, queueDir) + }) + return s.proposals +} + +// allowSubmission records a submission if the hourly budget allows it, or +// reports how long until it does. +func (p *channelProposalService) allowSubmission() (bool, time.Duration) { + p.rateMu.Lock() + defer p.rateMu.Unlock() + now := p.now() + cutoff := now.Add(-time.Hour) + i := 0 + for i < len(p.recent) && !p.recent[i].After(cutoff) { + i++ + } + p.recent = p.recent[i:] + if len(p.recent) >= p.limits.SubmissionsPerHour { + return false, p.recent[0].Add(time.Hour).Sub(now) + } + p.recent = append(p.recent, now) + return true, 0 +} + +// refundSubmission gives back the budget of a submission that was not queued. +func (p *channelProposalService) refundSubmission() { + p.rateMu.Lock() + defer p.rateMu.Unlock() + if n := len(p.recent); n > 0 { + p.recent = p.recent[:n-1] + } +} + +// approvedChannels returns the approved channels for GET /api/channels from a +// short-lived cache. It returns a fresh slice the caller may keep; the cached +// one is never handed out or mutated. +func (p *channelProposalService) approvedChannels(ctx context.Context) []ApprovedChannel { + if p == nil || p.db == nil { + return nil + } + p.approvedMu.Lock() + defer p.approvedMu.Unlock() + now := p.now() + if p.approved == nil || now.After(p.approved.expires) { + names, err := channelregistry.ListApprovedNames(ctx, p.db, channelregistry.MaxListed) + switch { + case err == nil: + chans := make([]ApprovedChannel, len(names)) + for i, n := range names { + chans[i] = ApprovedChannel{Name: n, Hash: n} + } + p.approved = &approvedSnapshot{channels: chans, builtAt: now, expires: now.Add(approvedCacheTTL)} + case p.approved == nil: + log.Printf("[channel-proposals] listing approved channels failed: %v", err) + return nil + default: + // Keep serving the last good snapshot; retry after another TTL. + log.Printf("[channel-proposals] listing approved channels failed, serving cached list: %v", err) + p.approved.expires = now.Add(approvedCacheTTL) + } + } + if len(p.approved.channels) == 0 { + return nil + } + out := make([]ApprovedChannel, len(p.approved.channels)) + copy(out, p.approved.channels) + return out +} + +// noteDecision invalidates the approved snapshot early for an approval or +// revocation result that completed after the snapshot was read, so the +// deciding admin sees the change without waiting out approvedCacheTTL. Anyone +// can poll a request id, so an older result, or the same result polled again +// once the list has been re-read, leaves the cache alone. Clock skew between +// the ingestor and the server only shifts this by the skew; the TTL still +// bounds staleness either way. +func (p *channelProposalService) noteDecision(st channelregistry.RequestStatus) { + if st.Status != channelregistry.RequestApproved && st.Status != channelregistry.RequestRevoked { + return + } + p.approvedMu.Lock() + defer p.approvedMu.Unlock() + if p.approved != nil && st.CompletedAt > p.approved.builtAt.UnixMilli() { + p.approved = nil + } +} + +// builtinNames returns the hashtag names the ingestor already decrypts +// through its config-derived keys, from the file it publishes in the queue +// directory. One stat per call; the file is parsed again only when it +// changed, and a bad version is logged once, not on every poll. A missing or +// unreadable file means none are known. +func (p *channelProposalService) builtinNames() map[string]bool { + if p == nil || p.queue == nil { + return nil + } + info, err := os.Stat(p.queue.BuiltinNamesPath()) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("[channel-proposals] reading built-in channel names failed: %v", err) + } + return nil + } + p.builtinMu.Lock() + defer p.builtinMu.Unlock() + if c := p.builtin; c != nil && c.mod.Equal(info.ModTime()) && c.size == info.Size() { + return c.names + } + names, err := p.queue.ReadBuiltinNames() + if err != nil { + log.Printf("[channel-proposals] reading built-in channel names failed: %v", err) + names = nil + } + p.builtin = &builtinNamesCache{mod: info.ModTime(), size: info.Size(), names: names} + return names +} + +func noStore(w http.ResponseWriter) { + w.Header().Set("Cache-Control", "no-store") +} + +func setRetryAfter(w http.ResponseWriter, d time.Duration) { + w.Header().Set("Retry-After", fmt.Sprintf("%d", int(math.Ceil(d.Seconds())))) +} + +// GET /api/channel-proposals/config +func (s *Server) handleChannelProposalConfig(w http.ResponseWriter, r *http.Request) { + noStore(w) + writeJSON(w, ChannelProposalConfigResponse{Enabled: s.channelProposals().enabled}) +} + +// POST /api/channel-proposals +func (s *Server) handleChannelProposalSubmit(w http.ResponseWriter, r *http.Request) { + p := s.channelProposals() + noStore(w) + if !p.enabled { + writeError(w, http.StatusForbidden, "channel suggestions are disabled") + return + } + var body ChannelProposalSubmitRequest + if err := decodeSingleJSONObject(http.MaxBytesReader(w, r.Body, maxProposalBodyBytes), &body); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + name, err := channelregistry.NormalizeName(body.Name) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + ok, wait := p.allowSubmission() + if !ok { + setRetryAfter(w, wait) + writeError(w, http.StatusTooManyRequests, "too many suggestions right now, please try again later") + return + } + cmd := channelregistry.Command{ + RequestID: channelregistry.NewID(), + Op: channelregistry.OpSubmit, + Name: name, + CreatedAt: p.now().UnixMilli(), + } + if !s.enqueueProposalCommand(w, p, cmd) { + p.refundSubmission() + } +} + +// decodeSingleJSONObject decodes exactly one JSON value with only known +// fields into v; trailing data other than whitespace is an error. +func decodeSingleJSONObject(r io.Reader, v interface{}) error { + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + if _, err := dec.Token(); err != io.EOF { + return errors.New("unexpected data after the JSON body") + } + return nil +} + +// enqueueProposalCommand writes cmd and answers 202, or answers the error. +func (s *Server) enqueueProposalCommand(w http.ResponseWriter, p *channelProposalService, cmd channelregistry.Command) bool { + if p.queue == nil { + writeError(w, http.StatusServiceUnavailable, "channel suggestions are unavailable") + return false + } + if err := p.queue.Enqueue(cmd, p.limits.MaxQueuedRequests); err != nil { + if errors.Is(err, channelregistry.ErrQueueFull) { + setRetryAfter(w, time.Minute) + writeError(w, http.StatusServiceUnavailable, err.Error()) + return false + } + log.Printf("[channel-proposals] enqueue %s failed: %v", cmd.Op, err) + writeError(w, http.StatusInternalServerError, "could not queue the request") + return false + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + json.NewEncoder(w).Encode(ChannelProposalAcceptedResponse{RequestID: cmd.RequestID}) + return true +} + +// GET /api/channel-proposals/requests/{requestId} +func (s *Server) handleChannelProposalRequest(w http.ResponseWriter, r *http.Request) { + p := s.channelProposals() + noStore(w) + id := mux.Vars(r)["requestId"] + if !channelregistry.ValidID(id) { + writeError(w, http.StatusBadRequest, "invalid request id") + return + } + if p.queue == nil { + writeError(w, http.StatusNotFound, channelregistry.ErrUnknownRequest.Error()) + return + } + st, err := p.queue.Lookup(id) + if err != nil { + if errors.Is(err, channelregistry.ErrUnknownRequest) { + writeError(w, http.StatusNotFound, err.Error()) + return + } + log.Printf("[channel-proposals] lookup %s failed: %v", id, err) + writeError(w, http.StatusInternalServerError, "could not read the request status") + return + } + p.noteDecision(st) + resp := ChannelProposalRequestResponse{RequestStatus: st} + if st.Proposal != nil { + resp.BuiltIn = p.builtinNames()[st.Proposal.Name] + } + writeJSON(w, resp) +} + +// GET /api/admin/channel-proposals?status=pending|approved|rejected|revoked +func (s *Server) handleAdminChannelProposals(w http.ResponseWriter, r *http.Request) { + p := s.channelProposals() + noStore(w) + status := r.URL.Query().Get("status") + if status != "" && !channelregistry.ValidStatus(status) { + writeError(w, http.StatusBadRequest, "status must be pending, approved, rejected or revoked") + return + } + resp := ChannelProposalListResponse{Proposals: []AdminChannelProposal{}, Enabled: p.enabled} + if p.db != nil { + limit := p.limits.MaxPending + p.limits.MaxApproved + p.limits.MaxQueuedRequests + list, err := channelregistry.ListProposals(r.Context(), p.db, status, limit) + if err != nil { + log.Printf("[channel-proposals] admin list failed: %v", err) + writeError(w, http.StatusInternalServerError, "could not list suggestions") + return + } + builtin := p.builtinNames() + for _, pr := range list { + resp.Proposals = append(resp.Proposals, AdminChannelProposal{Proposal: pr, BuiltIn: builtin[pr.Name]}) + } + } + writeJSON(w, resp) +} + +// loadAdminProposal validates id and loads the proposal, writing the +// 400/404/500 response itself and returning ok=false when the caller must +// not proceed. Shared by the decision handler below and the revoke handler. +func (s *Server) loadAdminProposal(w http.ResponseWriter, r *http.Request, p *channelProposalService, id string) (proposal channelregistry.Proposal, ok bool) { + if !channelregistry.ValidID(id) { + writeError(w, http.StatusBadRequest, "invalid suggestion id") + return channelregistry.Proposal{}, false + } + if p.db == nil { + writeError(w, http.StatusNotFound, "suggestion not found") + return channelregistry.Proposal{}, false + } + proposal, found, err := channelregistry.GetProposal(r.Context(), p.db, id) + if err != nil { + log.Printf("[channel-proposals] get %s failed: %v", id, err) + writeError(w, http.StatusInternalServerError, "could not read the suggestion") + return channelregistry.Proposal{}, false + } + if !found { + writeError(w, http.StatusNotFound, "suggestion not found") + return channelregistry.Proposal{}, false + } + return proposal, true +} + +// POST /api/admin/channel-proposals/{id}/approve and .../reject +func (s *Server) handleAdminChannelProposalDecision(op string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + p := s.channelProposals() + noStore(w) + id := mux.Vars(r)["id"] + if _, ok := s.loadAdminProposal(w, r, p, id); !ok { + return + } + s.enqueueProposalCommand(w, p, channelregistry.Command{ + RequestID: channelregistry.NewID(), + Op: op, + ProposalID: id, + CreatedAt: p.now().UnixMilli(), + }) + } +} + +// POST /api/admin/channel-proposals/{id}/revoke undoes a previous approval. +// +// Unlike approve/reject, this performs a synchronous precondition check +// before enqueueing anything: only a currently-approved proposal may be +// revoked, and a caller who asks to revoke something else gets an +// immediate 409 with no side effect at all (no command file is written). +// This is a deliberate, narrow exception to the rest of this package's +// "let the ingestor validate asynchronously" pattern — approve/reject +// always answer 202 and let a conflict surface later via polling +// GET /api/channel-proposals/requests/{requestId} — because the caller +// here specifically wants a synchronous, side-effect-free rejection for +// the common case of "it's not approved anymore" instead of having to +// poll a request that was never going to do anything. +// +// There is still a real TOCTOU window between this check and the ingestor +// actually applying the command: another admin could approve, reject or +// revoke the same proposal in the meantime. reviewChannelProposal in the +// ingestor (cmd/ingestor/channel_proposals.go) re-validates the status +// from scratch under its own transaction and is the true source of truth; +// the 409 here is only a best-effort fast-fail for the common case, not a +// substitute for that re-validation. +func (s *Server) handleAdminChannelProposalRevoke(w http.ResponseWriter, r *http.Request) { + p := s.channelProposals() + noStore(w) + id := mux.Vars(r)["id"] + proposal, ok := s.loadAdminProposal(w, r, p, id) + if !ok { + return + } + if proposal.Status != channelregistry.StatusApproved { + writeError(w, http.StatusConflict, "suggestion is not currently approved") + return + } + s.enqueueProposalCommand(w, p, channelregistry.Command{ + RequestID: channelregistry.NewID(), + Op: channelregistry.OpRevoke, + ProposalID: id, + CreatedAt: p.now().UnixMilli(), + }) +} diff --git a/cmd/server/channel_proposals_test.go b/cmd/server/channel_proposals_test.go new file mode 100644 index 000000000..d6f6a9161 --- /dev/null +++ b/cmd/server/channel_proposals_test.go @@ -0,0 +1,763 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/channelregistry" +) + +const strongTestKey = "proposal-admin-key-strong-enough" + +// Test fixture only: the production server never writes this table. The +// ingestor creates it (internal/dbschema) and applies every change. +const proposalTableDDL = `CREATE TABLE channel_proposals ( + id TEXT PRIMARY KEY, + name TEXT COLLATE BINARY NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN ('pending','approved','rejected','revoked')), + created_at INTEGER NOT NULL, + reviewed_at INTEGER NULL +)` + +type proposalServer struct { + t *testing.T + srv *Server + router *mux.Router + svc *channelProposalService + queue *channelregistry.Queue + clock time.Time +} + +func newProposalServer(t *testing.T, apiKey string, pc *channelregistry.Config, withTable bool) *proposalServer { + t.Helper() + db := setupTestDB(t) + seedTestData(t, db) + if withTable { + if _, err := db.conn.Exec(proposalTableDDL); err != nil { + t.Fatal(err) + } + } + cfg := &Config{Port: 3000, APIKey: apiKey, ChannelProposals: pc} + srv := NewServer(db, cfg, NewHub()) + ps := &proposalServer{t: t, srv: srv, clock: time.UnixMilli(1_700_000_000_000)} + srv.proposals = newChannelProposalService(cfg, db.conn, filepath.Join(t.TempDir(), channelregistry.QueueDirName)) + srv.proposals.now = func() time.Time { return ps.clock } + ps.svc = srv.proposals + ps.queue = srv.proposals.queue + ps.router = mux.NewRouter() + srv.RegisterRoutes(ps.router) + return ps +} + +func enabledConfig(mod func(*channelregistry.Config)) *channelregistry.Config { + on := true + c := &channelregistry.Config{Enabled: &on} + if mod != nil { + mod(c) + } + return c +} + +func (ps *proposalServer) do(method, path, body, key string) *httptest.ResponseRecorder { + ps.t.Helper() + var rd *bytes.Reader + if body != "" { + rd = bytes.NewReader([]byte(body)) + } else { + rd = bytes.NewReader(nil) + } + req := httptest.NewRequest(method, path, rd) + if key != "" { + req.Header.Set("X-API-Key", key) + } + w := httptest.NewRecorder() + ps.router.ServeHTTP(w, req) + return w +} + +func (ps *proposalServer) insert(id, name, status string) { + ps.t.Helper() + var reviewed interface{} + if status != channelregistry.StatusPending { + reviewed = ps.clock.UnixMilli() + } + if _, err := ps.srv.db.conn.Exec(`INSERT INTO channel_proposals (id, name, status, created_at, reviewed_at) VALUES (?, ?, ?, ?, ?)`, + id, name, status, ps.clock.UnixMilli(), reviewed); err != nil { + ps.t.Fatal(err) + } +} + +func decode[T any](t *testing.T, w *httptest.ResponseRecorder) T { + t.Helper() + var v T + if err := json.Unmarshal(w.Body.Bytes(), &v); err != nil { + t.Fatalf("decode %q: %v", w.Body.String(), err) + } + return v +} + +func TestChannelProposalConfigNeedsEnabledAndStrongKey(t *testing.T) { + off := false + cases := []struct { + name string + key string + pc *channelregistry.Config + want bool + }{ + {"enabled with strong key", strongTestKey, enabledConfig(nil), true}, + {"enabled with weak key", "test", enabledConfig(nil), false}, + {"enabled with short key", "short-key", enabledConfig(nil), false}, + {"enabled without key", "", enabledConfig(nil), false}, + {"disabled", strongTestKey, &channelregistry.Config{Enabled: &off}, false}, + {"not configured", strongTestKey, nil, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ps := newProposalServer(t, c.key, c.pc, true) + w := ps.do("GET", "/api/channel-proposals/config", "", "") + if w.Code != 200 { + t.Fatalf("status %d", w.Code) + } + if got := decode[ChannelProposalConfigResponse](t, w); got.Enabled != c.want { + t.Fatalf("enabled = %v, want %v", got.Enabled, c.want) + } + sub := ps.do("POST", "/api/channel-proposals", `{"name":"#Channel"}`, "") + if c.want && sub.Code != http.StatusAccepted { + t.Fatalf("submit = %d %s", sub.Code, sub.Body) + } + if !c.want && sub.Code != http.StatusForbidden { + t.Fatalf("disabled submit = %d, want 403", sub.Code) + } + }) + } +} + +func TestChannelProposalSubmitValidatesAndQueues(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + for _, body := range []string{`not json`, `{"name":""}`, `{"name":"#` + strings.Repeat("x", 40) + `"}`, `{"name":"#a\u0000b"}`} { + if w := ps.do("POST", "/api/channel-proposals", body, ""); w.Code != http.StatusBadRequest { + t.Errorf("body %s = %d, want 400", body, w.Code) + } + } + big := `{"name":"` + strings.Repeat("a", 2000) + `"}` + if w := ps.do("POST", "/api/channel-proposals", big, ""); w.Code != http.StatusBadRequest { + t.Errorf("oversized body = %d, want 400", w.Code) + } + + w := ps.do("POST", "/api/channel-proposals", `{"name":" MeshCore "}`, "") + if w.Code != http.StatusAccepted { + t.Fatalf("submit = %d %s", w.Code, w.Body) + } + id := decode[ChannelProposalAcceptedResponse](t, w).RequestID + if !channelregistry.ValidID(id) { + t.Fatalf("request id %q", id) + } + pending, err := ps.queue.Pending() + if err != nil || len(pending) != 1 { + t.Fatalf("queue = %+v, %v", pending, err) + } + cmd := pending[0].Command + if cmd.Op != channelregistry.OpSubmit || cmd.Name != "#MeshCore" || cmd.RequestID != id || cmd.CreatedAt != ps.clock.UnixMilli() { + t.Fatalf("queued command = %+v (case must be preserved)", cmd) + } +} + +func TestChannelProposalRateLimitAndQueueBound(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(func(c *channelregistry.Config) { + c.SubmissionsPerHour = 2 + c.MaxQueuedRequests = 3 + }), true) + submit := func(name string) *httptest.ResponseRecorder { + return ps.do("POST", "/api/channel-proposals", `{"name":"`+name+`"}`, "") + } + if submit("#a").Code != 202 || submit("#b").Code != 202 { + t.Fatal("first two submissions must pass") + } + w := submit("#c") + if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "3600" { + t.Fatalf("third = %d Retry-After=%q", w.Code, w.Header().Get("Retry-After")) + } + ps.clock = ps.clock.Add(time.Hour + time.Second) + if w := submit("#c"); w.Code != 202 { + t.Fatalf("after the window = %d", w.Code) + } + // Queue bound: 3 commands are waiting, the next one is refused with 503 + // and does not consume rate budget. + ps.clock = ps.clock.Add(2 * time.Hour) + if w := submit("#d"); w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Fatalf("queue full = %d", w.Code) + } + if n := len(ps.svc.recent); n != 0 { + t.Fatalf("refused submission kept its rate slot: %d", n) + } + if w := ps.do("POST", "/api/admin/channel-proposals/1111111111111111/approve", "", strongTestKey); w.Code != http.StatusNotFound { + t.Fatalf("approve unknown = %d", w.Code) + } +} + +func TestChannelProposalRequestStatus(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + // mux answers path tricks with a clean-path redirect before any handler + // runs; either way no file outside the queue is ever read. + if w := ps.do("GET", "/api/channel-proposals/requests/..%2F..%2Fetc", "", ""); w.Code == http.StatusOK { + t.Fatalf("traversal id served: %s", w.Body) + } + if w := ps.do("GET", "/api/channel-proposals/requests/NOT-HEX", "", ""); w.Code != http.StatusBadRequest { + t.Fatalf("bad id = %d", w.Code) + } + if w := ps.do("GET", "/api/channel-proposals/requests/0123456789abcdef", "", ""); w.Code != http.StatusNotFound { + t.Fatalf("unknown id = %d", w.Code) + } + id := decode[ChannelProposalAcceptedResponse](t, ps.do("POST", "/api/channel-proposals", `{"name":"#Status"}`, "")).RequestID + w := ps.do("GET", "/api/channel-proposals/requests/"+id, "", "") + if w.Code != 200 || w.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("queued lookup = %d %s", w.Code, w.Header().Get("Cache-Control")) + } + if st := decode[channelregistry.RequestStatus](t, w); st.Status != channelregistry.RequestQueued || st.Proposal != nil { + t.Fatalf("queued = %+v", st) + } + // What the ingestor writes back. + p := &channelregistry.Proposal{ID: id, Name: "#Status", Status: channelregistry.StatusPending, CreatedAt: ps.clock.UnixMilli()} + if err := ps.queue.Complete(channelregistry.Result{RequestID: id, Status: channelregistry.RequestPending, Proposal: p}); err != nil { + t.Fatal(err) + } + raw := ps.do("GET", "/api/channel-proposals/requests/"+id, "", "").Body.String() + for _, want := range []string{`"status":"pending"`, `"name":"#Status"`, `"createdAt":1700000000000`} { + if !strings.Contains(raw, want) { + t.Fatalf("pending body %s lacks %s", raw, want) + } + } + if strings.Contains(raw, "reviewedAt") || strings.Contains(raw, `"error"`) { + t.Fatalf("pending body %s must omit reviewedAt/error", raw) + } +} + +func TestAdminChannelProposalsRequireStrongKey(t *testing.T) { + cases := []struct { + name, configured, sent string + want int + }{ + {"no key configured", "", strongTestKey, http.StatusForbidden}, + {"missing header", strongTestKey, "", http.StatusUnauthorized}, + {"wrong key", strongTestKey, "wrong-key-wrong-key-wrong", http.StatusUnauthorized}, + {"weak configured key", "test", "test", http.StatusForbidden}, + {"correct key", strongTestKey, strongTestKey, http.StatusOK}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ps := newProposalServer(t, c.configured, enabledConfig(nil), true) + if w := ps.do("GET", "/api/admin/channel-proposals", "", c.sent); w.Code != c.want { + t.Fatalf("list = %d, want %d", w.Code, c.want) + } + ps.insert("abcdefabcdefabcd", "#x", channelregistry.StatusPending) + want := c.want + if want == http.StatusOK { + want = http.StatusAccepted + } + if w := ps.do("POST", "/api/admin/channel-proposals/abcdefabcdefabcd/approve", "", c.sent); w.Code != want { + t.Fatalf("approve = %d, want %d", w.Code, want) + } + }) + } +} + +func TestAdminChannelProposalListAndDecisions(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.insert("aaaaaaaaaaaaaaaa", "#pending", channelregistry.StatusPending) + ps.clock = ps.clock.Add(time.Second) + ps.insert("bbbbbbbbbbbbbbbb", "#approved", channelregistry.StatusApproved) + ps.insert("cccccccccccccccc", "#rejected", channelregistry.StatusRejected) + + all := decode[ChannelProposalListResponse](t, ps.do("GET", "/api/admin/channel-proposals", "", strongTestKey)) + if len(all.Proposals) != 3 || !all.Enabled { + t.Fatalf("list = %+v", all) + } + only := decode[ChannelProposalListResponse](t, ps.do("GET", "/api/admin/channel-proposals?status=approved", "", strongTestKey)) + if len(only.Proposals) != 1 || only.Proposals[0].Name != "#approved" || only.Proposals[0].ReviewedAt == nil { + t.Fatalf("approved filter = %+v", only) + } + if w := ps.do("GET", "/api/admin/channel-proposals?status=bogus", "", strongTestKey); w.Code != http.StatusBadRequest { + t.Fatalf("bad status filter = %d", w.Code) + } + + for _, op := range []string{"approve", "reject"} { + w := ps.do("POST", "/api/admin/channel-proposals/aaaaaaaaaaaaaaaa/"+op, "", strongTestKey) + if w.Code != http.StatusAccepted { + t.Fatalf("%s = %d %s", op, w.Code, w.Body) + } + if id := decode[ChannelProposalAcceptedResponse](t, w).RequestID; !channelregistry.ValidID(id) { + t.Fatalf("%s request id %q", op, id) + } + } + pending, _ := ps.queue.Pending() + if len(pending) != 2 || pending[0].Command.ProposalID != "aaaaaaaaaaaaaaaa" { + t.Fatalf("queued decisions = %+v", pending) + } + ops := map[string]bool{pending[0].Command.Op: true, pending[1].Command.Op: true} + if !ops[channelregistry.OpApprove] || !ops[channelregistry.OpReject] { + t.Fatalf("ops = %v", ops) + } + if w := ps.do("POST", "/api/admin/channel-proposals/not-hex/approve", "", strongTestKey); w.Code != http.StatusBadRequest { + t.Fatalf("bad id = %d", w.Code) + } + // Moderation stays available when public submissions are switched off. + off := false + closed := newProposalServer(t, strongTestKey, &channelregistry.Config{Enabled: &off}, true) + closed.insert("dddddddddddddddd", "#late", channelregistry.StatusPending) + if w := closed.do("POST", "/api/admin/channel-proposals/dddddddddddddddd/approve", "", strongTestKey); w.Code != http.StatusAccepted { + t.Fatalf("approve with submissions off = %d", w.Code) + } +} + +func TestChannelsListIncludesApprovedWithoutTraffic(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + + before := ps.do("GET", "/api/channels", "", "") + if before.Code != 200 || strings.Contains(before.Body.String(), "approvedChannels") { + t.Fatalf("empty list must omit approvedChannels: %s", before.Body) + } + baseline := decode[ChannelListResponse](t, before).Channels + + ps.insert("aaaaaaaaaaaaaaaa", "#QuietChannel", channelregistry.StatusApproved) + ps.insert("bbbbbbbbbbbbbbbb", "#Waiting", channelregistry.StatusPending) + ps.insert("cccccccccccccccc", "#Nope", channelregistry.StatusRejected) + ps.clock = ps.clock.Add(approvedCacheTTL + time.Second) + + resp := decode[ChannelListResponse](t, ps.do("GET", "/api/channels", "", "")) + want := []ApprovedChannel{{Name: "#QuietChannel", Hash: "#QuietChannel"}} + if !reflect.DeepEqual(resp.ApprovedChannels, want) { + t.Fatalf("approvedChannels = %+v, want %+v", resp.ApprovedChannels, want) + } + if len(resp.Channels) != len(baseline) { + t.Fatalf("channels changed: %d -> %d", len(baseline), len(resp.Channels)) + } + // Approved even though it never carried a message. + for _, ch := range resp.Channels { + if ch["name"] == "#QuietChannel" { + t.Fatal("precondition: #QuietChannel must have no traffic") + } + } +} + +// The approval shows up without waiting for the cache TTL once a request +// status read observes it. +func TestApprovedStatusInvalidatesCache(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.do("GET", "/api/channels", "", "") // warm the (empty) cache + ps.insert("aaaaaaaaaaaaaaaa", "#Fresh", channelregistry.StatusApproved) + reqID := channelregistry.NewID() + p := &channelregistry.Proposal{ID: "aaaaaaaaaaaaaaaa", Name: "#Fresh", Status: channelregistry.StatusApproved} + if err := ps.queue.Complete(channelregistry.Result{RequestID: reqID, Status: channelregistry.RequestApproved, Proposal: p, CompletedAt: ps.clock.Add(time.Millisecond).UnixMilli()}); err != nil { + t.Fatal(err) + } + if st := decode[channelregistry.RequestStatus](t, ps.do("GET", "/api/channel-proposals/requests/"+reqID, "", "")); st.Status != channelregistry.RequestApproved { + t.Fatalf("status = %+v", st) + } + resp := decode[ChannelListResponse](t, ps.do("GET", "/api/channels", "", "")) + if len(resp.ApprovedChannels) != 1 || resp.ApprovedChannels[0].Name != "#Fresh" { + t.Fatalf("approvedChannels after approval = %+v", resp.ApprovedChannels) + } +} + +// GET /api/channels must neither mutate the DB's cached channel slice nor +// hand the approved-channel cache to callers. +func TestChannelsListDoesNotMutateCaches(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.insert("aaaaaaaaaaaaaaaa", "#Shared", channelregistry.StatusApproved) + db := ps.srv.db + // An encrypted channel makes ?includeEncrypted=true append to the list. + if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) + VALUES ('EEFF', 'enc0000000000001', '2024-01-01T00:00:00Z', 1, 5, '{}', 'enc_AB')`); err != nil { + t.Fatal(err) + } + // The bug this guards against only shows when the cached slice has spare + // capacity, so pad the decrypted channels until it does. + var cached []map[string]interface{} + for i := 0; ; i++ { + db.channelsCacheMu.Lock() + db.channelsCacheRes = nil + db.channelsCacheMu.Unlock() + var err error + if cached, err = db.GetChannels(""); err != nil { + t.Fatal(err) + } + if cap(cached) > len(cached) { + break + } + if i == 8 { + t.Fatalf("precondition: cached slice never had spare capacity (len=cap=%d)", len(cached)) + } + if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) + VALUES ('EEFF', ?, '2024-01-01T00:00:00Z', 1, 5, '{"text":"a: b","sender":"a"}', ?)`, + "pad000000000000"+string(rune('a'+i)), "#pad"+string(rune('a'+i))); err != nil { + t.Fatal(err) + } + } + full := cached[:cap(cached)] + snapshot := make([]map[string]interface{}, len(full)) + for i, m := range full { + if m == nil { + continue + } + cp := make(map[string]interface{}, len(m)) + for k, v := range m { + cp[k] = v + } + snapshot[i] = cp + } + for i := 0; i < 3; i++ { + for _, q := range []string{"/api/channels", "/api/channels?includeEncrypted=true"} { + if w := ps.do("GET", q, "", ""); w.Code != 200 { + t.Fatalf("%s = %d", q, w.Code) + } + } + } + again, _ := db.GetChannels("") + if len(again) != len(cached) || !reflect.DeepEqual(full, cached[:cap(cached)]) { + t.Fatal("cached channel slice changed") + } + for i, m := range cached[:cap(cached)] { + if !reflect.DeepEqual(m, snapshot[i]) { + t.Fatalf("cached channel %d mutated: %v -> %v", i, snapshot[i], m) + } + } + a := ps.svc.approvedChannels(context.Background()) + a[0].Name = "#Tampered" + if b := ps.svc.approvedChannels(context.Background()); b[0].Name != "#Shared" { + t.Fatal("approvedChannels handed out its cached slice") + } +} + +// Rolling upgrade: a new server against a database whose ingestor has not +// created the table yet. Everything reads as empty, nothing fails, and the +// absence is not cached: once the table appears, approvals show up. +func TestChannelProposalsMissingTable(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), false) + + w := ps.do("GET", "/api/channels", "", "") + if w.Code != 200 || strings.Contains(w.Body.String(), "approvedChannels") { + t.Fatalf("channels = %d %s", w.Code, w.Body) + } + list := ps.do("GET", "/api/admin/channel-proposals", "", strongTestKey) + if list.Code != 200 || len(decode[ChannelProposalListResponse](t, list).Proposals) != 0 { + t.Fatalf("admin list = %d %s", list.Code, list.Body) + } + if w := ps.do("POST", "/api/admin/channel-proposals/aaaaaaaaaaaaaaaa/approve", "", strongTestKey); w.Code != http.StatusNotFound { + t.Fatalf("approve = %d", w.Code) + } + if w := ps.do("POST", "/api/channel-proposals", `{"name":"#Early"}`, ""); w.Code != http.StatusAccepted { + t.Fatalf("submit is queued for the ingestor even before the table exists: %d", w.Code) + } + + if _, err := ps.srv.db.conn.Exec(proposalTableDDL); err != nil { + t.Fatal(err) + } + ps.insert("aaaaaaaaaaaaaaaa", "#Later", channelregistry.StatusApproved) + ps.clock = ps.clock.Add(approvedCacheTTL + time.Second) + resp := decode[ChannelListResponse](t, ps.do("GET", "/api/channels", "", "")) + if len(resp.ApprovedChannels) != 1 || resp.ApprovedChannels[0].Name != "#Later" { + t.Fatalf("approvedChannels after the table appeared = %+v", resp.ApprovedChannels) + } +} + +// ── revoke ────────────────────────────────────────────────────────────── + +func queueFileNames(t *testing.T, ps *proposalServer) []string { + t.Helper() + entries, err := os.ReadDir(ps.queue.Dir()) + if err != nil { + if os.IsNotExist(err) { + return nil + } + t.Fatal(err) + } + var out []string + for _, e := range entries { + out = append(out, e.Name()) + } + return out +} + +func TestAdminChannelProposalRevokeRequiresStrongKey(t *testing.T) { + cases := []struct { + name, configured, sent string + want int + }{ + {"no key configured", "", strongTestKey, http.StatusForbidden}, + {"missing header", strongTestKey, "", http.StatusUnauthorized}, + {"wrong key", strongTestKey, "wrong-key-wrong-key-wrong", http.StatusUnauthorized}, + {"weak configured key", "test", "test", http.StatusForbidden}, + {"correct key", strongTestKey, strongTestKey, http.StatusAccepted}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ps := newProposalServer(t, c.configured, enabledConfig(nil), true) + ps.insert("abcdefabcdefabcd", "#x", channelregistry.StatusApproved) + if w := ps.do("POST", "/api/admin/channel-proposals/abcdefabcdefabcd/revoke", "", c.sent); w.Code != c.want { + t.Fatalf("revoke = %d, want %d", w.Code, c.want) + } + }) + } +} + +func TestAdminChannelProposalRevokeUnknownIDIs404WithNoQueueFile(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + before := queueFileNames(t, ps) + if w := ps.do("POST", "/api/admin/channel-proposals/ffffffffffffffff/revoke", "", strongTestKey); w.Code != http.StatusNotFound { + t.Fatalf("revoke unknown = %d", w.Code) + } + if got := queueFileNames(t, ps); !reflect.DeepEqual(got, before) { + t.Fatalf("revoke of an unknown id wrote a queue file: %v -> %v", before, got) + } + if w := ps.do("POST", "/api/admin/channel-proposals/not-hex/revoke", "", strongTestKey); w.Code != http.StatusBadRequest { + t.Fatalf("revoke bad id = %d", w.Code) + } +} + +// The synchronous, side-effect-free 409: pending, rejected and already- +// revoked proposals may not be revoked, and none of these writes a command +// file to the queue. +func TestAdminChannelProposalRevokeNonApprovedIs409WithNoQueueFile(t *testing.T) { + for _, status := range []string{channelregistry.StatusPending, channelregistry.StatusRejected, channelregistry.StatusRevoked} { + t.Run(status, func(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.insert("abcdefabcdefabcd", "#x", status) + before := queueFileNames(t, ps) + w := ps.do("POST", "/api/admin/channel-proposals/abcdefabcdefabcd/revoke", "", strongTestKey) + if w.Code != http.StatusConflict { + t.Fatalf("revoke of %s = %d, want 409", status, w.Code) + } + if got := queueFileNames(t, ps); !reflect.DeepEqual(got, before) { + t.Fatalf("409 revoke wrote a queue file: %v -> %v", before, got) + } + }) + } +} + +func TestAdminChannelProposalRevokeApprovedQueuesExactlyOneRevokeCommand(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.insert("abcdefabcdefabcd", "#x", channelregistry.StatusApproved) + w := ps.do("POST", "/api/admin/channel-proposals/abcdefabcdefabcd/revoke", "", strongTestKey) + if w.Code != http.StatusAccepted { + t.Fatalf("revoke = %d %s", w.Code, w.Body) + } + id := decode[ChannelProposalAcceptedResponse](t, w).RequestID + if !channelregistry.ValidID(id) { + t.Fatalf("request id %q", id) + } + pending, err := ps.queue.Pending() + if err != nil || len(pending) != 1 { + t.Fatalf("queue = %+v, %v", pending, err) + } + cmd := pending[0].Command + if cmd.Op != channelregistry.OpRevoke || cmd.ProposalID != "abcdefabcdefabcd" || cmd.RequestID != id { + t.Fatalf("queued command = %+v", cmd) + } +} + +// Once a channel is (simulated as) revoked in the DB, GET /api/channels no +// longer lists it, and a request-status read that observes the revocation +// invalidates the cache immediately rather than waiting out the TTL — +// mirroring TestApprovedStatusInvalidatesCache for approval. +func TestRevokedStatusInvalidatesApprovedCache(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.insert("aaaaaaaaaaaaaaaa", "#Fresh", channelregistry.StatusApproved) + ps.clock = ps.clock.Add(approvedCacheTTL + time.Second) + warm := decode[ChannelListResponse](t, ps.do("GET", "/api/channels", "", "")) + if len(warm.ApprovedChannels) != 1 { + t.Fatalf("precondition: #Fresh must be approved and cached: %+v", warm) + } + + // The ingestor revoked it directly in the DB (as RunOnce would after + // applying an OpRevoke command). + if _, err := ps.srv.db.conn.Exec(`UPDATE channel_proposals SET status = 'revoked', reviewed_at = ? WHERE id = ?`, + ps.clock.UnixMilli(), "aaaaaaaaaaaaaaaa"); err != nil { + t.Fatal(err) + } + reqID := channelregistry.NewID() + p := &channelregistry.Proposal{ID: "aaaaaaaaaaaaaaaa", Name: "#Fresh", Status: channelregistry.StatusRevoked} + if err := ps.queue.Complete(channelregistry.Result{RequestID: reqID, Status: channelregistry.RequestRevoked, Proposal: p, CompletedAt: ps.clock.Add(time.Millisecond).UnixMilli()}); err != nil { + t.Fatal(err) + } + // Still inside the cache TTL: without the early invalidation this would + // still read the stale, pre-revoke snapshot. + if st := decode[channelregistry.RequestStatus](t, ps.do("GET", "/api/channel-proposals/requests/"+reqID, "", "")); st.Status != channelregistry.RequestRevoked { + t.Fatalf("status = %+v", st) + } + resp := decode[ChannelListResponse](t, ps.do("GET", "/api/channels", "", "")) + if len(resp.ApprovedChannels) != 0 { + t.Fatalf("approvedChannels after revoke = %+v, want none", resp.ApprovedChannels) + } +} + +// TestOpenAPICompleteness (openapi_completeness_test.go) already asserts +// every /api/ HandleFunc/Handle route is covered by routeDescriptions() or +// the known-gaps allowlist; running the full suite (see the report) proves +// the new revoke route passes that gate without a gaps-file entry. + +// The server keeps its read-only contract: the queue directory is the only +// thing it writes, and only command files. +func TestChannelProposalServerWritesOnlyCommandFiles(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.do("POST", "/api/channel-proposals", `{"name":"#OnlyQueue"}`, "") + var n int + if err := ps.srv.db.conn.QueryRow(`SELECT COUNT(*) FROM channel_proposals`).Scan(&n); err != nil || n != 0 { + t.Fatalf("server wrote %d row(s), %v", n, err) + } + entries, err := os.ReadDir(ps.queue.Dir()) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if !strings.HasPrefix(e.Name(), "cmd-") { + t.Fatalf("unexpected file %s", e.Name()) + } + } +} + +// Anyone may poll a request id, so reading an approved/revoked result must +// not flush the approved-channel cache every time: only a result the current +// snapshot may predate (completed after it was built) invalidates it, once. +func TestRequestStatusPollsDoNotFlushApprovedCache(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + ps.insert("aaaaaaaaaaaaaaaa", "#A", channelregistry.StatusApproved) + ps.clock = ps.clock.Add(approvedCacheTTL + time.Second) + names := func() string { + var out []string + for _, c := range decode[ChannelListResponse](t, ps.do("GET", "/api/channels", "", "")).ApprovedChannels { + out = append(out, c.Name) + } + return strings.Join(out, ",") + } + if got := names(); got != "#A" { + t.Fatalf("precondition approved = %q", got) + } + poll := func(id string) { + t.Helper() + if w := ps.do("GET", "/api/channel-proposals/requests/"+id, "", ""); w.Code != http.StatusOK { + t.Fatalf("poll = %d %s", w.Code, w.Body) + } + } + complete := func(status string, completedAt time.Time) string { + t.Helper() + id := channelregistry.NewID() + p := &channelregistry.Proposal{ID: "aaaaaaaaaaaaaaaa", Name: "#A", Status: status} + if err := ps.queue.Complete(channelregistry.Result{RequestID: id, Status: status, Proposal: p, CompletedAt: completedAt.UnixMilli()}); err != nil { + t.Fatal(err) + } + return id + } + + // An old result (completed before the snapshot) never invalidates. + old := complete(channelregistry.RequestApproved, ps.clock.Add(-time.Minute)) + ps.insert("bbbbbbbbbbbbbbbb", "#B", channelregistry.StatusApproved) // changed behind the cache + for i := 0; i < 5; i++ { + poll(old) + ps.clock = ps.clock.Add(time.Millisecond) + if got := names(); got != "#A" { + t.Fatalf("poll %d of an old result flushed the cache: approved = %q", i, got) + } + } + + // A new result invalidates once; polling it again does not. + fresh := complete(channelregistry.RequestRevoked, ps.clock.Add(time.Millisecond)) + ps.clock = ps.clock.Add(2 * time.Millisecond) + poll(fresh) + if got := names(); got != "#A,#B" { + t.Fatalf("a new result must invalidate: approved = %q", got) + } + ps.insert("cccccccccccccccc", "#C", channelregistry.StatusApproved) + for i := 0; i < 5; i++ { + ps.clock = ps.clock.Add(time.Millisecond) + poll(fresh) + if got := names(); got != "#A,#B" { + t.Fatalf("repeated poll %d flushed the cache again: approved = %q", i, got) + } + } +} + +// Approving or revoking a name the ingestor already decrypts through its +// config-derived layer (built-in, rainbow table, hashChannels, channelKeys) +// changes nothing; the admin list and the submitter's status say so. +func TestChannelProposalBuiltinNamesAreMarked(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + if err := ps.queue.WriteBuiltinNames([]string{"#test", "#chat"}); err != nil { + t.Fatal(err) + } + ps.insert("aaaaaaaaaaaaaaaa", "#test", channelregistry.StatusPending) + ps.insert("bbbbbbbbbbbbbbbb", "#Test", channelregistry.StatusApproved) // different bytes: not built in + w := ps.do("GET", "/api/admin/channel-proposals", "", strongTestKey) + list := decode[ChannelProposalListResponse](t, w) + marks := map[string]bool{} + for _, p := range list.Proposals { + marks[p.Name] = p.BuiltIn + } + if len(marks) != 2 || !marks["#test"] || marks["#Test"] { + t.Fatalf("builtIn marks = %v (%s)", marks, w.Body) + } + if strings.Count(w.Body.String(), `"builtIn":true`) != 1 { + t.Fatalf("builtIn must be omitted when false: %s", w.Body) + } + + id := channelregistry.NewID() + p := &channelregistry.Proposal{ID: id, Name: "#chat", Status: channelregistry.StatusPending} + if err := ps.queue.Complete(channelregistry.Result{RequestID: id, Status: channelregistry.RequestPending, Proposal: p}); err != nil { + t.Fatal(err) + } + st := decode[ChannelProposalRequestResponse](t, ps.do("GET", "/api/channel-proposals/requests/"+id, "", "")) + if st.Status != channelregistry.RequestPending || !st.BuiltIn { + t.Fatalf("request status for a built-in name = %+v", st) + } + + // Without the file (an older ingestor) nothing is marked. + if err := os.Remove(ps.queue.BuiltinNamesPath()); err != nil { + t.Fatal(err) + } + list = decode[ChannelProposalListResponse](t, ps.do("GET", "/api/admin/channel-proposals", "", strongTestKey)) + for _, p := range list.Proposals { + if p.BuiltIn { + t.Fatalf("%s marked built in without a names file", p.Name) + } + } +} + +// The submit body is exactly one JSON object with known fields. +func TestChannelProposalSubmitRejectsTrailingDataAndUnknownFields(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + for _, body := range []string{ + `{"name":"#a"} {"name":"#b"}`, + `{"name":"#a"}garbage`, + `{"name":"#a","extra":1}`, + } { + if w := ps.do("POST", "/api/channel-proposals", body, ""); w.Code != http.StatusBadRequest { + t.Errorf("body %s = %d, want 400", body, w.Code) + } + } + if w := ps.do("POST", "/api/channel-proposals", "{\"name\":\"#ok\"}\n", ""); w.Code != http.StatusAccepted { + t.Fatalf("trailing whitespace must stay accepted: %d %s", w.Code, w.Body) + } +} + +// Invisible formatting characters are rejected at the API edge too. +func TestChannelProposalSubmitRejectsInvisibleCharacters(t *testing.T) { + ps := newProposalServer(t, strongTestKey, enabledConfig(nil), true) + for _, name := range []string{`#a\u200bb`, `#\ufeffab`, `#a\u00adb`, `#a\udb40\udc61b`, `#a\u2028b`} { + w := ps.do("POST", "/api/channel-proposals", `{"name":"`+name+`"}`, "") + if w.Code != http.StatusBadRequest { + t.Errorf("name %s = %d, want 400", name, w.Code) + } + } + if pending, _ := ps.queue.Pending(); len(pending) != 0 { + t.Fatalf("queued %d command(s) for invalid names", len(pending)) + } +} diff --git a/cmd/server/config.go b/cmd/server/config.go index f1660cbf0..c1c547c72 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/meshcore-analyzer/channelregistry" "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/geofilter" ) @@ -148,6 +149,11 @@ type Config struct { DBPath string `json:"dbPath"` ListLimits *ListLimitsConfig `json:"listLimits"` + // ChannelProposals configures publicly suggested hashtag channels + // (internal/channelregistry). Submissions open only when enabled AND a + // strong apiKey is set; the same block is read by the ingestor. + ChannelProposals *channelregistry.Config `json:"channelProposals,omitempty"` + // HashRegions mirrors the ingestor's region-scope config (same // config.json key). The server never derives HMAC keys from it — it // only needs the configured *names* to report which regions have diff --git a/cmd/server/go.mod b/cmd/server/go.mod index d54d923fd..8e7cc2464 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -60,3 +60,7 @@ replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue require github.com/meshcore-analyzer/regions v0.0.0 replace github.com/meshcore-analyzer/regions => ../../internal/regions + +require github.com/meshcore-analyzer/channelregistry v0.0.0 + +replace github.com/meshcore-analyzer/channelregistry => ../../internal/channelregistry diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 4f3880ab1..fcb1e168e 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -148,9 +148,19 @@ func routeDescriptions() map[string]routeMeta { }}, // Channels - "GET /api/channels": {Summary: "List channels", Description: "Returns known mesh channels with message counts.", Tag: "channels"}, + "GET /api/channels": {Summary: "List channels", Description: "Returns known mesh channels with message counts. approvedChannels ([{name, hash}]) lists the shared hashtag channels an administrator approved, even before they carry traffic; omitted when empty.", Tag: "channels"}, "GET /api/channels/{hash}/messages": {Summary: "Get channel messages", Description: "Returns messages for a specific channel.", Tag: "channels"}, + // Shared channel proposals (internal/channelregistry). The server only + // queues requests; the ingestor applies them. Timestamps are Unix ms. + "GET /api/channel-proposals/config": {Summary: "Channel suggestion availability", Description: "Returns {enabled}: whether public suggestions are open (requires channelProposals.enabled and a strong apiKey).", Tag: "channels"}, + "POST /api/channel-proposals": {Summary: "Suggest a public hashtag channel", Description: "Body {name}, exactly one JSON object without other fields. Only public hashtag channel names (at most 31 UTF-8 bytes including #, case preserved, no control, line-separator or invisible formatting characters) are accepted — never keys. Returns 202 {requestId}; 400 invalid body or name, 403 disabled, 429 rate limited, 503 queue full (both with Retry-After).", Tag: "channels"}, + "GET /api/channel-proposals/requests/{requestId}": {Summary: "Status of a suggestion or review request", Description: "Returns {status: queued|pending|approved|rejected|revoked|error, proposal: {id, name, status, createdAt, reviewedAt}, error, builtIn}. builtIn is true when the ingestor already decrypts the name through its built-in/config list. 404 when unknown or expired (24h).", Tag: "channels"}, + "GET /api/admin/channel-proposals": {Summary: "List channel suggestions", Description: "Returns {proposals, enabled}, newest first, bounded. A proposal whose name the ingestor already decrypts through its built-in/config list (rainbow table, hashChannels, channelKeys) carries builtIn: true. Optional status filter.", Tag: "admin", Auth: true, QueryParams: []paramMeta{{Name: "status", Description: "pending, approved, rejected or revoked", Type: "string"}}}, + "POST /api/admin/channel-proposals/{id}/approve": {Summary: "Approve a channel suggestion", Description: "Queues the approval and returns 202 {requestId}. Approved channels are decrypted by the ingestor and listed for everyone.", Tag: "admin", Auth: true}, + "POST /api/admin/channel-proposals/{id}/reject": {Summary: "Reject a channel suggestion", Description: "Queues the rejection and returns 202 {requestId}.", Tag: "admin", Auth: true}, + "POST /api/admin/channel-proposals/{id}/revoke": {Summary: "Revoke an approved channel suggestion", Description: "Undoes a previous approval: the ingestor stops decrypting the channel and it drops out of GET /api/channels' approvedChannels. Historical messages already decoded and stored are NOT deleted or hidden. Synchronous precondition check: 202 {requestId} only when the proposal is currently approved; 409 (no side effect, nothing queued) when it is not.", Tag: "admin", Auth: true}, + // Observers "GET /api/observers": {Summary: "List observers", Description: "Returns all known packet observers/gateways.", Tag: "observers"}, "GET /api/observers/{id}": {Summary: "Get observer detail", Tag: "observers"}, diff --git a/cmd/server/readonly_sql_literal_test.go b/cmd/server/readonly_sql_literal_test.go new file mode 100644 index 000000000..8a63b1e98 --- /dev/null +++ b/cmd/server/readonly_sql_literal_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "testing" +) + +// writeSQLPattern matches SQL that writes or changes a database. It is +// applied to the value of every string literal in non-test cmd/server +// source, so comments never match. +var writeSQLPattern = regexp.MustCompile(`(?is)\b(INSERT\s+(OR\s+\w+\s+)?INTO|REPLACE\s+INTO|UPDATE\s+(OR\s+\w+\s+)?[\w"]+\s+SET|DELETE\s+FROM|CREATE\s+(TEMP\w*\s+|UNIQUE\s+|VIRTUAL\s+)*(TABLE|INDEX|TRIGGER|VIEW)|DROP\s+(TABLE|INDEX|TRIGGER|VIEW)|ALTER\s+TABLE|VACUUM|REINDEX|ATTACH\s+DATABASE)\b`) + +// knownServerWriteSQL lists the write-SQL string literals cmd/server already +// had when this guard was added, per file. The count may only go down: a +// new write belongs in cmd/ingestor (#1283). Lower the number here when a +// site is removed. +// - ping_score_history.go: its own separate history database, not the +// shared one. +// - backup.go: VACUUM INTO writes a snapshot file, not the database. +// - openapi.go: prose mentioning VACUUM INTO. +// - hash_migrate.go, routes.go: pre-existing writes to the shared +// database, tracked separately. +var knownServerWriteSQL = map[string]int{ + "backup.go": 1, + "hash_migrate.go": 3, + "openapi.go": 1, + "ping_score_history.go": 15, + "routes.go": 3, +} + +// TestServerSourceHasNoNewWriteSQL guards the read-only server contract +// (#1283) at the source level: no new INSERT/UPDATE/DELETE/CREATE/DROP/ALTER +// (and the like) may appear in cmd/server, for example a shortcut that +// writes channel_proposals directly instead of queueing a command for the +// ingestor. +func TestServerSourceHasNoNewWriteSQL(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + got := map[string][]string{} + for _, name := range files { + if strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + ast.Inspect(f, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + v, err := strconv.Unquote(lit.Value) + if err != nil { + return true + } + if m := writeSQLPattern.FindString(v); m != "" { + got[name] = append(got[name], fset.Position(lit.Pos()).String()+": "+strings.Join(strings.Fields(m), " ")) + } + return true + }) + } + names := make([]string, 0, len(got)+len(knownServerWriteSQL)) + for name := range got { + names = append(names, name) + } + for name := range knownServerWriteSQL { + if _, ok := got[name]; !ok { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + hits, allowed := got[name], knownServerWriteSQL[name] + switch { + case len(hits) > allowed: + t.Errorf("%s has %d write-SQL literal(s), %d allowed — cmd/server is read-only, put writes in cmd/ingestor (#1283):\n %s", + name, len(hits), allowed, strings.Join(hits, "\n ")) + case len(hits) < allowed: + t.Errorf("%s now has %d write-SQL literal(s); lower knownServerWriteSQL[%q] from %d to %d", name, len(hits), name, allowed, len(hits)) + } + } +} + +// The pattern itself: writes match, reads and prose do not. +func TestWriteSQLPattern(t *testing.T) { + for _, s := range []string{ + "INSERT INTO channel_proposals (id) VALUES (?)", + "insert or replace into x values (1)", + "REPLACE INTO x VALUES (1)", + "UPDATE channel_proposals SET status = 'approved'", + "DELETE FROM channel_proposals", + "CREATE TABLE IF NOT EXISTS t (a)", + "CREATE UNIQUE INDEX i ON t(a)", + "CREATE TEMP TABLE t (a)", + "DROP TABLE t", + "ALTER TABLE t ADD COLUMN b", + "VACUUM", + "ATTACH DATABASE 'x' AS y", + } { + if !writeSQLPattern.MatchString(s) { + t.Errorf("must match %q", s) + } + } + for _, s := range []string{ + "SELECT id, name FROM channel_proposals WHERE status = ?", + "SELECT COUNT(*) FROM observations WHERE updated_at > ?", + "could not update the channel list", + "deleted_at", + "Update available", + } { + if writeSQLPattern.MatchString(s) { + t.Errorf("must not match %q", s) + } + } +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 5084fa394..efbaff8c9 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -18,6 +18,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/meshcore-analyzer/channelregistry" "github.com/meshcore-analyzer/geofilter" "github.com/meshcore-analyzer/packetpath" "github.com/meshcore-analyzer/prunequeue" @@ -57,6 +58,10 @@ type Server struct { // miss moves through handleStats. Nil in production. statsHook func(stage string) + // Shared channel proposals; built lazily from cfg/db (tests may preset). + proposals *channelProposalService + proposalsOnce sync.Once + // Guards s.cfg.GeoFilter — read by ingest/handler goroutines, written by PUT handler cfgMu sync.RWMutex @@ -321,6 +326,12 @@ func (s *Server) RegisterRoutes(r *mux.Router) { // runs the DELETE. /status reports completion. r.Handle("/api/admin/prune-geo-filter", s.requireAPIKey(http.HandlerFunc(s.handlePruneGeoFilter))).Methods("POST") r.Handle("/api/admin/prune-geo-filter/status", s.requireAPIKey(http.HandlerFunc(s.handlePruneGeoFilterStatus))).Methods("GET") + // Shared channel proposals: admin review uses the existing apiKey. The + // server only queues decisions; the ingestor applies them. + r.Handle("/api/admin/channel-proposals", s.requireAPIKey(http.HandlerFunc(s.handleAdminChannelProposals))).Methods("GET") + r.Handle("/api/admin/channel-proposals/{id}/approve", s.requireAPIKey(s.handleAdminChannelProposalDecision(channelregistry.OpApprove))).Methods("POST") + r.Handle("/api/admin/channel-proposals/{id}/reject", s.requireAPIKey(s.handleAdminChannelProposalDecision(channelregistry.OpReject))).Methods("POST") + r.Handle("/api/admin/channel-proposals/{id}/revoke", s.requireAPIKey(http.HandlerFunc(s.handleAdminChannelProposalRevoke))).Methods("POST") r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET") r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET") r.Handle("/api/backup", s.requireAPIKey(http.HandlerFunc(s.handleBackup))).Methods("GET") @@ -384,6 +395,9 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/resolve-hops", s.handleResolveHops).Methods("GET") r.HandleFunc("/api/channels/{hash}/messages", s.handleChannelMessages).Methods("GET") r.HandleFunc("/api/channels", s.handleChannels).Methods("GET") + r.HandleFunc("/api/channel-proposals/config", s.handleChannelProposalConfig).Methods("GET") + r.HandleFunc("/api/channel-proposals", s.handleChannelProposalSubmit).Methods("POST") + r.HandleFunc("/api/channel-proposals/requests/{requestId}", s.handleChannelProposalRequest).Methods("GET") r.HandleFunc("/api/known-channels", s.handleKnownChannels).Methods("GET") r.HandleFunc("/api/observers/metrics/summary", s.handleMetricsSummary).Methods("GET") r.HandleFunc("/api/observers/{id}/metrics", s.handleObserverMetrics).Methods("GET") @@ -3369,7 +3383,7 @@ func (s *Server) handleChannels(w http.ResponseWriter, r *http.Request) { channels = append(channels[:len(channels):len(channels)], encrypted...) } } - writeJSON(w, ChannelListResponse{Channels: channels}) + writeJSON(w, ChannelListResponse{Channels: channels, ApprovedChannels: s.channelProposals().approvedChannels(r.Context())}) return } if s.store != nil { @@ -3380,7 +3394,7 @@ func (s *Server) handleChannels(w http.ResponseWriter, r *http.Request) { // spare capacity of the shared backing array. channels = append(channels[:len(channels):len(channels)], s.store.GetEncryptedChannels(region)...) } - writeJSON(w, ChannelListResponse{Channels: channels}) + writeJSON(w, ChannelListResponse{Channels: channels, ApprovedChannels: s.channelProposals().approvedChannels(r.Context())}) return } writeJSON(w, ChannelListResponse{Channels: []map[string]interface{}{}}) diff --git a/cmd/server/types.go b/cmd/server/types.go index b3d81c7df..f530fdb4b 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -1421,6 +1421,16 @@ type ChannelResp struct { type ChannelListResponse struct { Channels []map[string]interface{} `json:"channels"` + // ApprovedChannels are the shared hashtag channels an administrator + // approved, listed even before they carry any traffic. Omitted when empty. + ApprovedChannels []ApprovedChannel `json:"approvedChannels,omitempty"` +} + +// ApprovedChannel is one shared hashtag channel. Hash equals Name: decrypted +// hashtag traffic is stored under the channel name (see db.GetChannels). +type ApprovedChannel struct { + Name string `json:"name"` + Hash string `json:"hash"` } type ChannelMessagesResponse struct { diff --git a/config.example.json b/config.example.json index 5bb7cbf0f..b0282f606 100644 --- a/config.example.json +++ b/config.example.json @@ -229,6 +229,15 @@ "#bookclub", "#shtf" ], + "channelProposals": { + "enabled": false, + "maxPending": 100, + "maxApproved": 128, + "maxQueuedRequests": 256, + "retentionDays": 30, + "submissionsPerHour": 20 + }, + "_comment_channelProposals": "Shared hashtag channels: visitors suggest public hashtag channels (names only, never keys) on the Channels page and an administrator approves or rejects them at #/channels?view=proposals with the apiKey. Submissions open only when enabled is true AND apiKey is strong (16+ characters, not a placeholder). Approved channels are decrypted by the ingestor and listed for everyone, even before they carry traffic; they stay active if enabled is later set to false. Limits: maxPending/maxApproved bound the table, maxQueuedRequests bounds the request queue next to the database, submissionsPerHour is a global sliding window, and rejected or unreviewed suggestions are deleted after retentionDays. Read by both server and ingestor.", "healthThresholds": { "infraDegradedHours": 24, "infraSilentHours": 72, diff --git a/docs/api-spec.md b/docs/api-spec.md index e56806d14..a2481e45c 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -35,6 +35,12 @@ - [GET /api/observers/:id/analytics](#get-apiobserversidanalytics) - [GET /api/channels](#get-apichannels) - [GET /api/channels/:hash/messages](#get-apichannelshashmessages) +- [GET /api/channel-proposals/config](#get-apichannel-proposalsconfig) +- [POST /api/channel-proposals](#post-apichannel-proposals) +- [GET /api/channel-proposals/requests/:requestId](#get-apichannel-proposalsrequestsrequestid) +- [GET /api/admin/channel-proposals](#get-apiadminchannel-proposals) +- [POST /api/admin/channel-proposals/:id/approve and /reject](#post-apiadminchannel-proposalsidapprove-and-reject) +- [POST /api/admin/channel-proposals/:id/revoke](#post-apiadminchannel-proposalsidrevoke) - [GET /api/analytics/rf](#get-apianalyticsrf) - [GET /api/analytics/topology](#get-apianalyticstopology) - [GET /api/analytics/channels](#get-apianalyticschannels) @@ -1216,6 +1222,11 @@ List decoded channels with message counts. "messageCount": number, "lastActivity": string (ISO) } + ], + // Shared hashtag channels approved by the administrator, listed even + // before they carry traffic. Omitted when there are none. hash == name. + "approvedChannels"?: [ + { "name": string, "hash": string } ] } ``` @@ -1263,6 +1274,100 @@ Messages for a specific channel. --- +## Shared channel proposals + +Visitors suggest public hashtag channels; the administrator approves or rejects them with the existing `apiKey`. The server is read-only: it validates each request and writes it to a bounded file queue next to the database, and the ingestor applies it. An accepted request answers `202 { "requestId": string }` right away; poll the request status for the outcome. Validation errors answer `4xx` before anything is queued, and revoke also checks the stored status first and answers `409` when the proposal is not approved (see below). Timestamps are Unix epoch **milliseconds**. + +A proposal: + +```jsonc +{ + "id": string, // 16 hex characters + "name": string, // "#Channel", case preserved, <= 31 UTF-8 bytes + "status": "pending" | "approved" | "rejected" | "revoked", + "createdAt": number, // ms + "reviewedAt"?: number // ms, once reviewed (also set when revoked) +} +``` + +In `GET /api/admin/channel-proposals` each proposal may also carry `"builtIn": true` (see [Built-in names](#built-in-names)). + +State machine: `pending` → `approved` or `rejected` (admin decision); `approved` → `revoked` (admin revoke, see below); `revoked` → `pending` by suggesting the same name again (never auto-approved — see POST /api/channel-proposals). `rejected` is terminal: suggesting a rejected name again reports the earlier rejection and does not reopen it, until retention deletes the rejected row `channelProposals.retentionDays` (default 30) days after the review; from then on the name can be suggested afresh. This keeps a rejected name from being pushed back into the review queue over and over. + +### Built-in names + +The ingestor already decrypts every name in its config-derived key list: the built-in keys, the rainbow table `channel-rainbow.json` (about 320 common names such as `#test`, `#chat` and `#general`), `hashChannels` and `channelKeys`. That list wins over approved suggestions, so approving such a name adds nothing (its traffic was already decrypted and listed once it has traffic) and revoking it does not stop decryption. The ingestor publishes those hashtag names to `builtin-channels.json` in the request queue directory at startup and after each `SIGHUP` reload; the server marks matching proposals with `"builtIn": true` in the admin list and in the request status, and the UI says so. An ingestor that predates this file simply leaves nothing marked. + +## GET /api/channel-proposals/config + +```json +{ "enabled": true } +``` + +`enabled` is true only when `channelProposals.enabled` is set and `apiKey` is strong. + +## POST /api/channel-proposals + +Body `{ "name": "#Channel" }` (the `#` is optional; surrounding spaces are trimmed). Only public hashtag channel names are accepted, never keys. + +| Status | Meaning | +|--------|---------| +| `202` | `{ "requestId": string }` | +| `400` | Invalid body or name. The body must be exactly one JSON object with only `name` (unknown fields and trailing data are refused). The name must not be empty or over 31 bytes, and must not contain control characters, the line/paragraph separators U+2028/U+2029, direction overrides, or other invisible formatting characters (Unicode category Cf, such as U+200B, U+FEFF, U+00AD and the tag characters U+E0000–U+E007F; the zero-width joiner U+200D is allowed, and variation selectors are not Cf, so emoji sequences work). The firmware only limits the length; the character rule is CoreScope's own, so two different channels cannot look identical. | +| `403` | Suggestions disabled | +| `429` | `submissionsPerHour` reached; `Retry-After` in seconds | +| `503` | Request queue full; `Retry-After` in seconds | + +## GET /api/channel-proposals/requests/:requestId + +```jsonc +{ + "status": "queued" | "pending" | "approved" | "rejected" | "revoked" | "error", + "proposal"?: Proposal, // once the ingestor has processed the request + "error"?: string, // with status "error" + "builtIn"?: true // the proposal's name is a built-in name (see above) +} +``` + +A duplicate suggestion reports the existing proposal and its status (for a rejected name: `rejected`, until retention). `revoked` is the outcome of a revoke request. `404` when the id is unknown or older than 24 hours. + +## GET /api/admin/channel-proposals + +Requires `X-API-Key`. Optional `?status=pending|approved|rejected|revoked`. Newest first, bounded. + +```jsonc +{ "proposals": [Proposal & { "builtIn"?: true }], "enabled": boolean } +``` + +## POST /api/admin/channel-proposals/:id/approve and /reject + +Requires `X-API-Key`. `202 { "requestId": string }`, `404` for an unknown proposal. Only pending proposals change; repeating the stored decision is harmless, and a contradicting one (reject after approve) ends with status `error`. + +Missing or wrong key: `401`. No key configured, or a weak one: `403`. + +## POST /api/admin/channel-proposals/:id/revoke + +Requires `X-API-Key`. Undoes a previous approval: the ingestor stops decrypting the channel (unless the name is also in its config-derived list — the rainbow table, `channelKeys` or `hashChannels`, see [Built-in names](#built-in-names) — in which case that key keeps it decrypting) and it drops out of `GET /api/channels`' `approvedChannels`. + +Unlike approve/reject, this endpoint checks synchronously, before queuing anything: + +| Status | Meaning | +|--------|---------| +| `202` | `{ "requestId": string }` — the proposal was approved; the revoke is queued | +| `400` | Invalid suggestion id | +| `404` | Unknown proposal | +| `409` | The proposal is not currently approved (pending, rejected, or already revoked) — **nothing is queued** | + +Missing or wrong key: `401`. No key configured, or a weak one: `403`. + +There is a real race between the `409` check and the ingestor actually applying the command: another admin could approve, reject or revoke the same proposal in between. The ingestor re-validates the status from scratch when it applies the command and is the true source of truth; the synchronous `409` here is only a best-effort fast-fail for the common case, not a guarantee. When the race is lost, the request ends with status `error` and an error such as `suggestion is not approved (it is pending)`. + +**Historical messages are not affected.** Revoking a channel only removes its decryption key going forward — messages the ingestor already decoded and stored while the channel was approved stay exactly as they are and remain visible on the Channels page. There is no mechanism (and none is planned as part of this) to hide or delete previously-decoded messages when a channel is revoked. + +**Retention.** A revoked proposal's row is not deleted at revoke time — only its status changes, keeping `reviewedAt` as the audit timestamp of when it was revoked. It is removed later by the same retention sweep that prunes rejected proposals, once `reviewedAt` is older than `channelProposals.retentionDays` (see [Configuration](user-guide/configuration.md#shared-channel-suggestions)). Approved rows are still never pruned. + +--- + ## GET /api/analytics/rf RF signal analytics. diff --git a/docs/user-guide/channels.md b/docs/user-guide/channels.md index a518463e3..46c895dc1 100644 --- a/docs/user-guide/channels.md +++ b/docs/user-guide/channels.md @@ -57,6 +57,39 @@ The `hashChannels` config lists channel names that CoreScope should try to match CoreScope computes the hash of each name and matches incoming packets to identify which channel they belong to. +## Shared channels + +When the administrator enables `channelProposals` (see [Configuration](configuration.md#shared-channel-suggestions)), the **Add Channel** dialog gets a **Suggest a Shared Channel** section. Anyone can suggest a public hashtag channel there. Only the name is sent, never a key, and the name is case-sensitive: `#MeshCore` and `#meshcore` are different channels. + +Names can be at most 31 bytes including the `#`. That is the firmware's limit (the name is stored in a 32-byte field with a terminating NUL). Any language or emoji is fine; control characters, line breaks, text-direction overrides and other invisible formatting characters (such as zero-width spaces or soft hyphens) are refused. The firmware itself only limits the length; the character rule is CoreScope's own, so that two different channels can never look the same. Emoji built with the zero-width joiner or variation selectors are allowed. + +An administrator reviews suggestions at `#/channels?view=proposals`: + +1. Enter the `apiKey`. It is kept only in the tab's memory and is forgotten on reload or with **Lock**. +2. Approve or reject each pending suggestion. + +Approved channels are decrypted by the ingestor from then on and appear for everyone under **Network** with a **Shared** label, even before they carry any messages. Shared channels have no remove button for a regular visitor, because they are not stored in your browser. + +A rejected name stays rejected: suggesting it again shows the earlier rejection instead of starting a new review, until the rejection is deleted by retention (`channelProposals.retentionDays`, 30 days by default, counted from the review). After that it can be suggested again. + +### Built-in names + +CoreScope already decrypts about 320 common hashtag channels out of the box (the rainbow table `channel-rainbow.json`, for example `#test`, `#chat` and `#general`), plus any names in `hashChannels` or `channelKeys`. Suggesting or approving one of these changes nothing, and removing it does not stop it from being decrypted. The review dialog labels such suggestions **Built in: already decrypted**, the Remove confirmation says the channel keeps being decrypted, and a visitor who suggests one is told so. + +### Revoking an approved channel + +An administrator can undo a previous approval from the same **Pending / Approved / Rejected / Revoked** review dialog: switch to the **Approved** tab and click **Remove** on a channel. A confirmation prompt names the channel before anything happens — nothing is sent until you confirm. + +Removing a channel: + +- Stops the ingestor from decrypting new traffic on it (unless it is a [built-in name](#built-in-names) — in the rainbow table or configured via `channelKeys`/`hashChannels` in `config.json` — in which case that key keeps working; see [Configuration](configuration.md#channel-decryption)). +- Takes it out of everyone's **Network** section going forward. +- Does **not** delete or hide any messages that were already decoded and shown while it was approved — those stay on the Channels page exactly as before. Revoking only affects future traffic. + +A removed channel shows up under the **Revoked** tab. Suggesting the same name again later starts a fresh review from **Pending** — it is never auto-approved just because it was approved before. + +The local **Add Channel** tools (PSK channels, **Monitor Hashtag Channel**) are unchanged and still only affect your browser. + ## Region filter Channels respect the region filter. Select a region to see only messages captured by observers in that area. diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index b84df8eb7..89b61eaed 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -123,6 +123,23 @@ See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background. See [Channels](channels.md) for details. +### Shared channel suggestions + +`channelProposals` lets visitors suggest public hashtag channels that an administrator approves for everyone. The server and the ingestor read the same block. + +| Field | Default | Description | +|-------|---------|-------------| +| `enabled` | `false` | Opens public suggestions. Only takes effect with a strong `apiKey` (16+ characters, not a placeholder). | +| `maxPending` | `100` | Suggestions waiting for review. Further suggestions are refused until some are reviewed. | +| `maxApproved` | `128` | Shared channels that can be approved. | +| `maxQueuedRequests` | `256` | Requests waiting for the ingestor in the queue directory next to the database. | +| `retentionDays` | `30` | Rejected and never-reviewed suggestions are deleted after this many days. Approved channels are kept. Until its rejection is deleted, a rejected name cannot be suggested again. | +| `submissionsPerHour` | `20` | Global limit on new suggestions per hour. It is global rather than per visitor, because behind a reverse proxy every visitor can share one address. | + +Approved channels stay decrypted and listed when `enabled` is later set to `false`, and survive restarts and `SIGHUP` reloads. A key configured in `channelKeys` for the same name takes priority, and so does the rainbow table (`channel-rainbow.json`): approving or revoking one of those built-in names changes nothing, and the review dialog marks them (see [Channels](channels.md#built-in-names)). + +An administrator can also revoke a previously approved channel (see [Channels](channels.md#revoking-an-approved-channel)) — this undoes the decryption going forward but never deletes or hides messages already decoded while it was approved. A revoked row is retained and pruned by the same `retentionDays` rule as a rejected one (counted from when it was revoked, not when it was first submitted); an approved row is still never pruned. Revoking introduces no new configuration of its own — it reuses the `maxPending`/`maxApproved`/`retentionDays`/`submissionsPerHour` limits above. + ## Map defaults ```json diff --git a/internal/channelregistry/builtin.go b/internal/channelregistry/builtin.go new file mode 100644 index 000000000..0db604df4 --- /dev/null +++ b/internal/channelregistry/builtin.go @@ -0,0 +1,98 @@ +package channelregistry + +import ( + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "sort" + "strings" +) + +// BuiltinNamesFileName is the file, inside the queue directory, where the +// ingestor publishes the hashtag channel names its config-derived key layer +// already decrypts: the built-in keys, the rainbow table +// (channel-rainbow.json, ~320 common names such as #test and #chat), +// hashChannels and channelKeys. That layer wins over approved suggestions, so +// approving such a name adds nothing and revoking it does not stop its +// traffic from being decrypted. The read-only server uses the file to say so +// in the admin list and in a submitter's request status. +const BuiltinNamesFileName = "builtin-channels.json" + +// MaxBuiltinNames bounds the published set. The shipped rainbow table has +// about 320 names; operators can add more through config. +const MaxBuiltinNames = 4096 + +// maxBuiltinNamesBytes bounds how much of the file a reader parses: +// MaxBuiltinNames names of at most MaxNameBytes bytes, JSON-quoted. +const maxBuiltinNamesBytes = MaxBuiltinNames * (MaxNameBytes*6 + 4) + +type builtinNamesFile struct { + Names []string `json:"names"` +} + +// BuiltinNamesPath returns the path of the builtin names file. +func (q *Queue) BuiltinNamesPath() string { + return filepath.Join(q.dir, BuiltinNamesFileName) +} + +// WriteBuiltinNames atomically replaces the published set with the distinct +// '#' names in names (other keys, such as "Public" or PSK channels, cannot be +// suggested and are left out), sorted and capped at MaxBuiltinNames. +func (q *Queue) WriteBuiltinNames(names []string) error { + seen := make(map[string]struct{}, len(names)) + out := make([]string, 0, len(names)) + for _, n := range names { + if len(n) < 2 || !strings.HasPrefix(n, "#") { + continue + } + if _, ok := seen[n]; ok { + continue + } + seen[n] = struct{}{} + out = append(out, n) + } + sort.Strings(out) + if len(out) > MaxBuiltinNames { + out = out[:MaxBuiltinNames] + } + q.mu.Lock() + defer q.mu.Unlock() + if err := os.MkdirAll(q.dir, 0o755); err != nil { + return err + } + return q.writeAtomic(q.BuiltinNamesPath(), builtinNamesFile{Names: out}) +} + +// ReadBuiltinNames returns the published set. A missing file (an ingestor +// that has not written it yet) is an empty set, not an error. +func (q *Queue) ReadBuiltinNames() (map[string]bool, error) { + f, err := os.Open(q.BuiltinNamesPath()) + if err != nil { + if os.IsNotExist(err) { + return map[string]bool{}, nil + } + return nil, err + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, maxBuiltinNamesBytes+1)) + if err != nil { + return nil, err + } + if len(b) > maxBuiltinNamesBytes { + return nil, errors.New("builtin channel names file is too large") + } + var file builtinNamesFile + if err := json.Unmarshal(b, &file); err != nil { + return nil, err + } + if len(file.Names) > MaxBuiltinNames { + file.Names = file.Names[:MaxBuiltinNames] + } + set := make(map[string]bool, len(file.Names)) + for _, n := range file.Names { + set[n] = true + } + return set, nil +} diff --git a/internal/channelregistry/config.go b/internal/channelregistry/config.go new file mode 100644 index 000000000..4c4caa6d4 --- /dev/null +++ b/internal/channelregistry/config.go @@ -0,0 +1,75 @@ +package channelregistry + +import "time" + +// Defaults for the limits. Every limit is bounded so the queue, the table and +// the per-request status files cannot grow without bound. +const ( + DefaultMaxPending = 100 + DefaultMaxApproved = 128 + DefaultMaxQueuedRequests = 256 + DefaultRetentionDays = 30 + DefaultSubmissionsPerHour = 20 + DefaultRequestTTL = 24 * time.Hour + + // maxResultFactor bounds the number of kept result files relative to the + // queue size, so a burst of finished requests cannot fill the directory. + maxResultFactor = 4 +) + +// Config is the "channelProposals" block in config.json. The server and the +// ingestor read the same block. Zero or negative values fall back to the +// defaults. +type Config struct { + // Enabled opens public submissions. It only takes effect when a strong + // admin apiKey is configured too. Approved channels stay active and + // visible when this is later turned off. + Enabled *bool `json:"enabled,omitempty"` + MaxPending int `json:"maxPending,omitempty"` + MaxApproved int `json:"maxApproved,omitempty"` + MaxQueuedRequests int `json:"maxQueuedRequests,omitempty"` + RetentionDays int `json:"retentionDays,omitempty"` + SubmissionsPerHour int `json:"submissionsPerHour,omitempty"` +} + +// Limits are the resolved limits, with defaults applied. +type Limits struct { + MaxPending int + MaxApproved int + MaxQueuedRequests int + MaxResults int + RetentionDays int + SubmissionsPerHour int + RequestTTL time.Duration +} + +// SubmissionsRequested reports whether the operator switched submissions on. +// Callers must additionally require a strong admin API key. +func (c *Config) SubmissionsRequested() bool { + return c != nil && c.Enabled != nil && *c.Enabled +} + +// Limits resolves the configured limits. Safe on a nil *Config. +func (c *Config) Limits() Limits { + var in Config + if c != nil { + in = *c + } + l := Limits{ + MaxPending: orDefault(in.MaxPending, DefaultMaxPending), + MaxApproved: orDefault(in.MaxApproved, DefaultMaxApproved), + MaxQueuedRequests: orDefault(in.MaxQueuedRequests, DefaultMaxQueuedRequests), + RetentionDays: orDefault(in.RetentionDays, DefaultRetentionDays), + SubmissionsPerHour: orDefault(in.SubmissionsPerHour, DefaultSubmissionsPerHour), + RequestTTL: DefaultRequestTTL, + } + l.MaxResults = l.MaxQueuedRequests * maxResultFactor + return l +} + +func orDefault(v, def int) int { + if v <= 0 { + return def + } + return v +} diff --git a/internal/channelregistry/go.mod b/internal/channelregistry/go.mod new file mode 100644 index 000000000..b22116a58 --- /dev/null +++ b/internal/channelregistry/go.mod @@ -0,0 +1,3 @@ +module github.com/meshcore-analyzer/channelregistry + +go 1.22 diff --git a/internal/channelregistry/name.go b/internal/channelregistry/name.go new file mode 100644 index 000000000..f18e69d46 --- /dev/null +++ b/internal/channelregistry/name.go @@ -0,0 +1,123 @@ +// Package channelregistry is the shared contract for publicly suggested +// hashtag channels: name validation, limits, the request/result file queue +// between the read-only server and the writer-owning ingestor, and the +// read-only SQL helpers both processes use. +// +// Write path (see cmd/ingestor/channel_proposals.go): the server never +// touches SQLite for writes. It validates a request, writes one atomic +// command file into the queue directory next to the database, and the +// ingestor applies it and writes a result file the server reads back. +package channelregistry + +import ( + "errors" + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +// MaxNameBytes is the longest channel name the firmware can hold, in bytes, +// including the leading '#'. MeshCore stores a channel name in +// ChannelDetails.name (char[32]) via StrHelper::strncpy, which keeps room for +// the terminating NUL, so anything past 31 bytes would be silently truncated +// on a radio and derive a different key than the one we derive here. +const MaxNameBytes = 31 + +// Name validation errors. The messages are shown to users as-is. +var ( + ErrNameEmpty = errors.New("channel name is empty") + ErrNameTooLong = fmt.Errorf("channel name must be at most %d bytes including the #", MaxNameBytes) + ErrNameInvalidUTF8 = errors.New("channel name is not valid UTF-8") + ErrNameControl = errors.New("channel name contains control or direction-override characters") + ErrNameSpace = errors.New("channel name must not start or end with a space") + ErrNameInvisible = errors.New("channel name contains invisible formatting characters") +) + +// NormalizeName turns user input into the exact channel name whose bytes the +// hashtag key is derived from (SHA-256("#name")[:16]). +// +// It mirrors how the ingestor treats hashChannels entries: surrounding +// whitespace is trimmed and a missing '#' is added. Nothing else is changed — +// case and Unicode form are kept byte-for-byte, because the firmware derives +// the key from the exact bytes, so "#Test" and "#test" are different channels. +// +// The firmware imposes no character set (it only limits the length), so +// neither do we beyond what could never be a working or honest name: invalid +// UTF-8, control characters (a NUL would truncate the name on a radio) and the +// line/paragraph separators U+2028/U+2029, bidirectional overrides and other +// invisible formatting characters (which make a name display as something it +// is not, or two different channels look identical), and a body that starts +// or ends with whitespace. The invisible-character rule is CoreScope's own +// presentation rule, not a firmware one; the frontend and the ingestor apply +// the same rule. +func NormalizeName(raw string) (string, error) { + s := strings.TrimSpace(raw) + if !utf8.ValidString(s) { + return "", ErrNameInvalidUTF8 + } + if !strings.HasPrefix(s, "#") { + s = "#" + s + } + body := s[1:] + if body == "" { + return "", ErrNameEmpty + } + if len(s) > MaxNameBytes { + return "", ErrNameTooLong + } + for _, r := range s { + if unicode.IsControl(r) || isBidiControl(r) || isLineBreakSeparator(r) { + return "", ErrNameControl + } + if isInvisibleFormat(r) { + return "", ErrNameInvisible + } + } + first, _ := utf8.DecodeRuneInString(body) + last, _ := utf8.DecodeLastRuneInString(body) + if unicode.IsSpace(first) || unicode.IsSpace(last) { + return "", ErrNameSpace + } + return s, nil +} + +// isBidiControl reports the explicit directional formatting characters +// (Unicode Bidi_Control). Other format characters such as the zero-width +// joiner stay allowed: emoji sequences need them. +func isBidiControl(r rune) bool { + switch { + case r == 0x061C, r == 0x200E, r == 0x200F: + return true + case r >= 0x202A && r <= 0x202E: + return true + case r >= 0x2066 && r <= 0x2069: + return true + } + return false +} + +// isLineBreakSeparator reports U+2028 LINE SEPARATOR and U+2029 PARAGRAPH +// SEPARATOR. They are not Cf (they are Zl/Zp), but a line break inside a +// channel name breaks every single-line place the name is shown. +func isLineBreakSeparator(r rune) bool { + return r == 0x2028 || r == 0x2029 +} + +// isInvisibleFormat reports Unicode format characters (category Cf) that +// render as nothing: zero-width space U+200B, BOM U+FEFF, soft hyphen U+00AD, +// the tag characters U+E0000–U+E007F and the like. They would let two +// different channel names (different keys) look identical. The exceptions +// are what emoji need: ZERO WIDTH JOINER U+200D and the variation selectors +// U+FE00–U+FE0F and U+E0100–U+E01EF (those are Mn, not Cf, today; they are +// listed so the rule stays explicit if that ever changes). Emoji tag +// sequences, such as the subdivision flags, are rejected with the tags. +func isInvisibleFormat(r rune) bool { + switch { + case r == 0x200D: + return false + case r >= 0xFE00 && r <= 0xFE0F, r >= 0xE0100 && r <= 0xE01EF: + return false + } + return unicode.Is(unicode.Cf, r) +} diff --git a/internal/channelregistry/name_test.go b/internal/channelregistry/name_test.go new file mode 100644 index 000000000..efb29142b --- /dev/null +++ b/internal/channelregistry/name_test.go @@ -0,0 +1,152 @@ +package channelregistry + +import ( + "errors" + "strings" + "testing" +) + +func TestNormalizeNameAcceptsAndPreservesCase(t *testing.T) { + cases := map[string]string{ + "#MeshCore": "#MeshCore", + "MeshCore": "#MeshCore", // missing '#' is added, like hashChannels + " #wardriving ": "#wardriving", + "#Test": "#Test", // case preserved: #Test and #test are different channels + "#test": "#test", + "#København": "#København", // non-ASCII is allowed; the firmware has no charset + "#my channel": "#my channel", + "##double": "##double", + "#\U0001f468\u200d\U0001f469\u200d\U0001f467": "#\U0001f468\u200d\U0001f469\u200d\U0001f467", // zero-width joiners in emoji stay allowed + "#" + strings.Repeat("a", 30): "#" + strings.Repeat("a", 30), // exactly 31 bytes + } + for in, want := range cases { + got, err := NormalizeName(in) + if err != nil { + t.Errorf("NormalizeName(%q) error %v, want %q", in, err, want) + continue + } + if got != want { + t.Errorf("NormalizeName(%q) = %q, want %q", in, got, want) + } + } +} + +func TestNormalizeNameRejects(t *testing.T) { + cases := []struct { + in string + want error + }{ + {"", ErrNameEmpty}, + {" ", ErrNameEmpty}, + {"#", ErrNameEmpty}, + {"#" + strings.Repeat("a", 31), ErrNameTooLong}, // 32 bytes: the firmware would truncate it + {"#" + strings.Repeat("æ", 16), ErrNameTooLong}, // 16 runes but 33 bytes: the limit is bytes + {"#0123456789abcdef0123456789abcdef", ErrNameTooLong}, // a pasted PSK is not a hashtag name + {"#bad\x00name", ErrNameControl}, + {"#tab\tname", ErrNameControl}, + {"#rtl\u202eevil", ErrNameControl}, + {"#iso\u2066late", ErrNameControl}, + {"# leading", ErrNameSpace}, + {"#\u00a0leading-nbsp", ErrNameSpace}, // trailing whitespace is trimmed; a leading one after # is not + {"#\xff\xfe", ErrNameInvalidUTF8}, + } + for _, c := range cases { + got, err := NormalizeName(c.in) + if !errors.Is(err, c.want) { + t.Errorf("NormalizeName(%q) = %q, %v; want error %v", c.in, got, err, c.want) + } + } +} + +func TestConfigLimitsDefaultsAndOverrides(t *testing.T) { + var nilCfg *Config + l := nilCfg.Limits() + if l.MaxPending != 100 || l.MaxApproved != 128 || l.MaxQueuedRequests != 256 || + l.RetentionDays != 30 || l.SubmissionsPerHour != 20 || l.RequestTTL.Hours() != 24 { + t.Fatalf("defaults = %+v", l) + } + if l.MaxResults != 256*maxResultFactor { + t.Fatalf("MaxResults = %d", l.MaxResults) + } + if nilCfg.SubmissionsRequested() { + t.Fatal("nil config must not request submissions") + } + on := true + c := &Config{Enabled: &on, MaxPending: 5, MaxApproved: -1, SubmissionsPerHour: 3} + l = c.Limits() + if l.MaxPending != 5 || l.MaxApproved != 128 || l.SubmissionsPerHour != 3 { + t.Fatalf("overrides = %+v", l) + } + if !c.SubmissionsRequested() { + t.Fatal("enabled config must request submissions") + } +} + +// Invisible formatting characters (Unicode category Cf) and the line/paragraph +// separators make two different channels look identical or break the layout, +// so they are rejected. This is our own presentation rule: the firmware only +// limits the length. ZWJ and variation selectors stay allowed for emoji. +func TestNormalizeNameRejectsInvisibleFormatCharacters(t *testing.T) { + cases := []struct { + name string + r string + want error + }{ + {"soft hyphen U+00AD", "\u00ad", ErrNameInvisible}, + {"Arabic number sign U+0600", "\u0600", ErrNameInvisible}, + {"Mongolian vowel separator U+180E", "\u180e", ErrNameInvisible}, + {"zero width space U+200B", "\u200b", ErrNameInvisible}, + {"zero width non-joiner U+200C", "\u200c", ErrNameInvisible}, + {"word joiner U+2060", "\u2060", ErrNameInvisible}, + {"invisible times U+2062", "\u2062", ErrNameInvisible}, + {"BOM / ZWNBSP U+FEFF", "\ufeff", ErrNameInvisible}, + {"interlinear annotation U+FFF9", "\ufff9", ErrNameInvisible}, + {"language tag U+E0001", "\U000e0001", ErrNameInvisible}, + {"tag space U+E0020", "\U000e0020", ErrNameInvisible}, + {"tag latin small a U+E0061", "\U000e0061", ErrNameInvisible}, + {"cancel tag U+E007F", "\U000e007f", ErrNameInvisible}, + {"line separator U+2028", "\u2028", ErrNameControl}, + {"paragraph separator U+2029", "\u2029", ErrNameControl}, + // Bidi controls are Cf too; they keep their more specific error. + {"right-to-left override U+202E", "\u202e", ErrNameControl}, + {"left-to-right mark U+200E", "\u200e", ErrNameControl}, + } + for _, c := range cases { + inputs := []string{"#a" + c.r + "b", "#" + c.r + "ab", "#ab" + c.r} + if c.r == "\u2028" || c.r == "\u2029" { + // Trailing separators are whitespace and are trimmed like any + // other trailing whitespace (hashChannels does the same). + inputs = inputs[:2] + } + for _, in := range inputs { + got, err := NormalizeName(in) + if !errors.Is(err, c.want) { + t.Errorf("%s: NormalizeName(%q) = %q, %v; want %v", c.name, in, got, err, c.want) + } + } + } +} + +func TestNormalizeNameTrimsTrailingLineSeparator(t *testing.T) { + if got, err := NormalizeName("#ab\u2028"); err != nil || got != "#ab" { + t.Fatalf("NormalizeName(trailing U+2028) = %q, %v; want \"#ab\"", got, err) + } +} + +func TestNormalizeNameAllowsEmojiJoinersAndVariationSelectors(t *testing.T) { + for _, in := range []string{ + "#\U0001f3f3\ufe0f\u200d\U0001f308", // rainbow flag: VS16 + ZWJ + "#\u2764\ufe0f", // red heart: VS16 + "#\u2764\ufe0e", // text-style heart: VS15 + "#\U0001f469\u200d\U0001f4bb", // woman technologist: ZWJ + "#a\ufe00b", // VS1 + "#\u845b\U000e0100", // ideographic variation selector VS17 + "#\U0001f441\ufe0f\u200d\U0001f5e8\ufe0f", // eye in speech bubble + "#mesh\u200dcore", // a bare ZWJ between letters is still allowed + } { + got, err := NormalizeName(in) + if err != nil || got != in { + t.Errorf("NormalizeName(%q) = %q, %v; want it unchanged", in, got, err) + } + } +} diff --git a/internal/channelregistry/queue.go b/internal/channelregistry/queue.go new file mode 100644 index 000000000..68576f9b1 --- /dev/null +++ b/internal/channelregistry/queue.go @@ -0,0 +1,357 @@ +package channelregistry + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// QueueDirName is the directory, next to the SQLite database, that holds the +// command and result files. Keeping it beside the database gives it the same +// volume, backup and permission story as the prune queue (internal/prunequeue). +// +// Layout: +// +// cmd-.json written by the server, one per request +// result-.json written by the ingestor after applying it; the +// ingestor then removes the command file +// .tmp-* in-flight writes, renamed into place atomically +const QueueDirName = "channel-proposal-requests" + +const ( + cmdPrefix = "cmd-" + resultPrefix = "result-" + tmpPrefix = ".tmp-" + fileSuffix = ".json" + + // staleTmpAge is how old an orphaned temp file (from a writer that died + // between create and rename) must be before Prune removes it. + staleTmpAge = time.Hour +) + +// Queue errors. +var ( + ErrQueueFull = errors.New("the suggestion queue is full, please try again later") + ErrInvalidID = errors.New("invalid request id") + ErrInvalidCommand = errors.New("invalid command") + ErrUnknownRequest = errors.New("unknown or expired request") +) + +// QueueDir returns the queue directory for a database path. +func QueueDir(dbPath string) string { + return filepath.Join(filepath.Dir(dbPath), QueueDirName) +} + +// NewID returns a random 16-hex-character id. Ids double as file names and as +// proposal ids, so they are random rather than time-based. +func NewID() string { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return fmt.Sprintf("%016x", time.Now().UnixNano()) + } + return hex.EncodeToString(b[:]) +} + +// ValidID accepts lowercase hex ids of 16 to 64 characters. Anything else is +// rejected before it can reach a file path. +func ValidID(id string) bool { + if len(id) < 16 || len(id) > 64 { + return false + } + for i := 0; i < len(id); i++ { + c := id[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// Queue is the file queue in one directory. The mutex only serializes callers +// inside one process; the server and the ingestor coordinate through atomic +// renames, never through shared locks. +type Queue struct { + dir string + mu sync.Mutex +} + +// NewQueue returns a queue rooted at dir. The directory is created on first +// write. +func NewQueue(dir string) *Queue { + return &Queue{dir: dir} +} + +// Dir returns the queue directory. +func (q *Queue) Dir() string { return q.dir } + +// QueuedCommand is one command file. Err is set when the file could not be +// read or parsed; the ingestor discards such files. +type QueuedCommand struct { + Path string + Command Command + Err error +} + +func (q *Queue) cmdPath(id string) string { + return filepath.Join(q.dir, cmdPrefix+id+fileSuffix) +} + +func (q *Queue) resultPath(id string) string { + return filepath.Join(q.dir, resultPrefix+id+fileSuffix) +} + +// Enqueue writes cmd atomically, refusing when maxQueued commands are already +// waiting. +func (q *Queue) Enqueue(cmd Command, maxQueued int) error { + if !ValidID(cmd.RequestID) { + return ErrInvalidID + } + switch cmd.Op { + case OpSubmit: + if cmd.Name == "" { + return ErrInvalidCommand + } + case OpApprove, OpReject, OpRevoke: + if !ValidID(cmd.ProposalID) { + return ErrInvalidCommand + } + default: + return ErrInvalidCommand + } + q.mu.Lock() + defer q.mu.Unlock() + if err := os.MkdirAll(q.dir, 0o755); err != nil { + return err + } + names, err := q.listNames(cmdPrefix) + if err != nil { + return err + } + if len(names) >= maxQueued { + return ErrQueueFull + } + return q.writeAtomic(q.cmdPath(cmd.RequestID), cmd) +} + +// Pending returns the waiting commands in submission order (createdAt, then +// request id). +func (q *Queue) Pending() ([]QueuedCommand, error) { + names, err := q.listNames(cmdPrefix) + if err != nil { + return nil, err + } + out := make([]QueuedCommand, 0, len(names)) + for _, name := range names { + p := filepath.Join(q.dir, name) + qc := QueuedCommand{Path: p} + b, err := os.ReadFile(p) + switch { + case err != nil: + qc.Err = err + case json.Unmarshal(b, &qc.Command) != nil: + qc.Err = ErrInvalidCommand + case !ValidID(qc.Command.RequestID) || p != q.cmdPath(qc.Command.RequestID): + qc.Err = ErrInvalidID + } + out = append(out, qc) + } + sort.SliceStable(out, func(i, j int) bool { + a, b := out[i].Command, out[j].Command + if a.CreatedAt != b.CreatedAt { + return a.CreatedAt < b.CreatedAt + } + return a.RequestID < b.RequestID + }) + return out, nil +} + +// Complete records the outcome of a command, then removes the command file. +// The result is written first: a crash between the two steps leaves both +// files, and the command is simply applied again, which the ingestor makes +// idempotent. +func (q *Queue) Complete(res Result) error { + if !ValidID(res.RequestID) { + return ErrInvalidID + } + q.mu.Lock() + defer q.mu.Unlock() + if err := os.MkdirAll(q.dir, 0o755); err != nil { + return err + } + if err := q.writeAtomic(q.resultPath(res.RequestID), res); err != nil { + return err + } + if err := os.Remove(q.cmdPath(res.RequestID)); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// Discard removes a command file that cannot be processed. +func (q *Queue) Discard(path string) error { + if filepath.Dir(path) != filepath.Clean(q.dir) || !strings.HasPrefix(filepath.Base(path), cmdPrefix) { + return ErrInvalidCommand + } + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// Lookup returns the public state of a request. +func (q *Queue) Lookup(id string) (RequestStatus, error) { + if !ValidID(id) { + return RequestStatus{}, ErrInvalidID + } + b, err := os.ReadFile(q.resultPath(id)) + if err == nil { + var res Result + if err := json.Unmarshal(b, &res); err != nil { + return RequestStatus{}, err + } + return RequestStatus{Status: res.Status, Proposal: res.Proposal, Error: res.Error, CompletedAt: res.CompletedAt}, nil + } + if !os.IsNotExist(err) { + return RequestStatus{}, err + } + if _, err := os.Stat(q.cmdPath(id)); err == nil { + return RequestStatus{Status: RequestQueued}, nil + } else if !os.IsNotExist(err) { + return RequestStatus{}, err + } + return RequestStatus{}, ErrUnknownRequest +} + +// Prune removes result files older than ttl, the oldest results beyond +// maxResults, and orphaned temp files. Commands are never pruned here: the +// ingestor answers expired commands with an error result instead, so a +// polling client always learns what happened. +func (q *Queue) Prune(now time.Time, ttl time.Duration, maxResults int) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + entries, err := os.ReadDir(q.dir) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + type aged struct { + path string + mod time.Time + } + var results []aged + removed := 0 + for _, e := range entries { + if e.IsDir() { + continue + } + info, err := e.Info() + if err != nil { + continue + } + p := filepath.Join(q.dir, e.Name()) + switch { + case strings.HasPrefix(e.Name(), tmpPrefix): + if now.Sub(info.ModTime()) > staleTmpAge && os.Remove(p) == nil { + removed++ + } + case strings.HasPrefix(e.Name(), resultPrefix) && strings.HasSuffix(e.Name(), fileSuffix): + if now.Sub(info.ModTime()) > ttl { + if os.Remove(p) == nil { + removed++ + } + continue + } + results = append(results, aged{p, info.ModTime()}) + } + } + if maxResults > 0 && len(results) > maxResults { + sort.Slice(results, func(i, j int) bool { return results[i].mod.Before(results[j].mod) }) + for _, r := range results[:len(results)-maxResults] { + if os.Remove(r.path) == nil { + removed++ + } + } + } + return removed, nil +} + +func (q *Queue) listNames(prefix string) ([]string, error) { + entries, err := os.ReadDir(q.dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var out []string + for _, e := range entries { + if e.IsDir() { + continue + } + if name := e.Name(); strings.HasPrefix(name, prefix) && strings.HasSuffix(name, fileSuffix) { + out = append(out, name) + } + } + return out, nil +} + +// writeAtomic writes v as JSON to a temp file in the same directory, syncs +// it, renames it into place and syncs the directory, so a reader never sees a +// partial file and a crash never leaves a truncated one under the final name. +func (q *Queue) writeAtomic(path string, v interface{}) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + f, err := os.CreateTemp(q.dir, tmpPrefix+"*") + if err != nil { + return err + } + tmp := f.Name() + if _, err := f.Write(b); err != nil { + f.Close() + os.Remove(tmp) + return err + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return err + } + if err := os.Chmod(tmp, 0o644); err != nil { + os.Remove(tmp) + return err + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return err + } + syncDir(filepath.Dir(path)) + return nil +} + +// syncDir fsyncs a directory so a rename into it survives a power loss, not +// just a process crash. Best effort: some filesystems and platforms refuse to +// sync a directory, and the file itself is already synced. +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + d.Sync() + d.Close() +} diff --git a/internal/channelregistry/queue_test.go b/internal/channelregistry/queue_test.go new file mode 100644 index 000000000..2af148cea --- /dev/null +++ b/internal/channelregistry/queue_test.go @@ -0,0 +1,241 @@ +package channelregistry + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func submit(id, name string, at int64) Command { + return Command{RequestID: id, Op: OpSubmit, Name: name, CreatedAt: at} +} + +func TestValidIDRejectsPathTricks(t *testing.T) { + for _, bad := range []string{"", "abc", "../../etc/passwd", "0123456789ABCDEF", "0123456789abcdeg", "0123456789abcdef/.."} { + if ValidID(bad) { + t.Errorf("ValidID(%q) = true", bad) + } + } + id := NewID() + if !ValidID(id) { + t.Fatalf("NewID() = %q is not valid", id) + } + if NewID() == id { + t.Fatal("NewID must be random") + } +} + +func TestEnqueueLookupCompleteLifecycle(t *testing.T) { + q := NewQueue(filepath.Join(t.TempDir(), QueueDirName)) + id := NewID() + if _, err := q.Lookup(id); !errors.Is(err, ErrUnknownRequest) { + t.Fatalf("unknown request lookup = %v", err) + } + if err := q.Enqueue(submit(id, "#Test", 1), 10); err != nil { + t.Fatal(err) + } + st, err := q.Lookup(id) + if err != nil || st.Status != RequestQueued { + t.Fatalf("queued lookup = %+v, %v", st, err) + } + pending, err := q.Pending() + if err != nil || len(pending) != 1 || pending[0].Err != nil || pending[0].Command.Name != "#Test" { + t.Fatalf("pending = %+v, %v", pending, err) + } + p := &Proposal{ID: id, Name: "#Test", Status: StatusPending, CreatedAt: 1} + if err := q.Complete(Result{RequestID: id, Status: RequestPending, Proposal: p, CompletedAt: 2}); err != nil { + t.Fatal(err) + } + st, err = q.Lookup(id) + if err != nil || st.Status != RequestPending || st.Proposal == nil || st.Proposal.Name != "#Test" { + t.Fatalf("completed lookup = %+v, %v", st, err) + } + if pending, _ := q.Pending(); len(pending) != 0 { + t.Fatalf("command file not removed: %+v", pending) + } + // Completing again (crash replay) is harmless. + if err := q.Complete(Result{RequestID: id, Status: RequestPending, Proposal: p, CompletedAt: 3}); err != nil { + t.Fatalf("second complete: %v", err) + } +} + +func TestEnqueueBoundedAndValidated(t *testing.T) { + q := NewQueue(filepath.Join(t.TempDir(), QueueDirName)) + for i := 0; i < 3; i++ { + if err := q.Enqueue(submit(NewID(), "#c", int64(i)), 3); err != nil { + t.Fatalf("enqueue %d: %v", i, err) + } + } + if err := q.Enqueue(submit(NewID(), "#c", 9), 3); !errors.Is(err, ErrQueueFull) { + t.Fatalf("4th enqueue = %v, want ErrQueueFull", err) + } + if err := q.Enqueue(submit("nothex", "#c", 1), 10); !errors.Is(err, ErrInvalidID) { + t.Fatalf("bad id = %v", err) + } + if err := q.Enqueue(Command{RequestID: NewID(), Op: OpApprove, ProposalID: "x"}, 10); !errors.Is(err, ErrInvalidCommand) { + t.Fatalf("bad proposal id = %v", err) + } + if err := q.Enqueue(Command{RequestID: NewID(), Op: "drop"}, 10); !errors.Is(err, ErrInvalidCommand) { + t.Fatalf("bad op = %v", err) + } + // No temp files survive a successful write. + entries, _ := os.ReadDir(q.Dir()) + for _, e := range entries { + if e.Name()[0] == '.' { + t.Fatalf("leftover temp file %s", e.Name()) + } + } +} + +func TestPendingOrderAndMalformed(t *testing.T) { + q := NewQueue(filepath.Join(t.TempDir(), QueueDirName)) + a, b := "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb" + if err := q.Enqueue(submit(b, "#second", 20), 10); err != nil { + t.Fatal(err) + } + if err := q.Enqueue(submit(a, "#first", 10), 10); err != nil { + t.Fatal(err) + } + bad := filepath.Join(q.Dir(), "cmd-cccccccccccccccc.json") + if err := os.WriteFile(bad, []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + pending, err := q.Pending() + if err != nil { + t.Fatal(err) + } + if len(pending) != 3 || pending[0].Err == nil { + t.Fatalf("malformed file should sort first (createdAt 0) with an error: %+v", pending) + } + if pending[1].Command.RequestID != a || pending[2].Command.RequestID != b { + t.Fatalf("order = %s, %s", pending[1].Command.RequestID, pending[2].Command.RequestID) + } + if err := q.Discard(pending[0].Path); err != nil { + t.Fatal(err) + } + if err := q.Discard(filepath.Join(t.TempDir(), "cmd-x.json")); err == nil { + t.Fatal("Discard outside the queue dir must fail") + } +} + +func TestPruneTTLCapAndTempFiles(t *testing.T) { + q := NewQueue(filepath.Join(t.TempDir(), QueueDirName)) + now := time.Now() + ids := []string{"1111111111111111", "2222222222222222", "3333333333333333", "4444444444444444"} + for i, id := range ids { + if err := q.Complete(Result{RequestID: id, Status: RequestPending, CompletedAt: int64(i)}); err != nil { + t.Fatal(err) + } + // Oldest first: 30h, 3h, 2h, 1h ago. + age := time.Duration(len(ids)-i) * time.Hour + if i == 0 { + age = 30 * time.Hour + } + p := filepath.Join(q.Dir(), resultPrefix+id+fileSuffix) + if err := os.Chtimes(p, now.Add(-age), now.Add(-age)); err != nil { + t.Fatal(err) + } + } + tmp := filepath.Join(q.Dir(), tmpPrefix+"orphan") + if err := os.WriteFile(tmp, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(tmp, now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + removed, err := q.Prune(now, 24*time.Hour, 2) + if err != nil { + t.Fatal(err) + } + // 1 expired + 1 over the cap of 2 + 1 orphaned temp file. + if removed != 3 { + t.Fatalf("removed = %d, want 3", removed) + } + for _, id := range ids[:2] { + if _, err := q.Lookup(id); !errors.Is(err, ErrUnknownRequest) { + t.Errorf("%s should be pruned, lookup = %v", id, err) + } + } + for _, id := range ids[2:] { + if _, err := q.Lookup(id); err != nil { + t.Errorf("%s should be kept, lookup = %v", id, err) + } + } + if _, err := os.Stat(tmp); !os.IsNotExist(err) { + t.Fatal("orphaned temp file not removed") + } +} + +// The ingestor publishes the hashtag names its config-derived key layer +// (built-in, rainbow table, hashChannels, channelKeys) already decrypts, so +// the read-only server can tell an administrator that approving or revoking +// such a name changes nothing. +func TestBuiltinNamesRoundTrip(t *testing.T) { + q := NewQueue(filepath.Join(t.TempDir(), QueueDirName)) + names, err := q.ReadBuiltinNames() + if err != nil || len(names) != 0 { + t.Fatalf("missing file must read as empty: %v, %v", names, err) + } + in := []string{"#test", "Public", "#Test", "#test", "", "#chat", "psk:abc"} + if err := q.WriteBuiltinNames(in); err != nil { + t.Fatal(err) + } + names, err = q.ReadBuiltinNames() + if err != nil { + t.Fatal(err) + } + want := map[string]bool{"#Test": true, "#chat": true, "#test": true} + if len(names) != len(want) { + t.Fatalf("names = %v, want only the distinct hashtag names %v", names, want) + } + for n := range want { + if !names[n] { + t.Fatalf("names = %v, missing %q", names, n) + } + } + // A rewrite replaces the set. + if err := q.WriteBuiltinNames([]string{"#only"}); err != nil { + t.Fatal(err) + } + names, _ = q.ReadBuiltinNames() + if len(names) != 1 || !names["#only"] { + t.Fatalf("after rewrite names = %v", names) + } + // The file never counts as a command or a result. + if pending, err := q.Pending(); err != nil || len(pending) != 0 { + t.Fatalf("builtin names file must not be a command: %v, %v", pending, err) + } + if _, err := q.Prune(time.Now().Add(1000*time.Hour), time.Hour, 1); err != nil { + t.Fatal(err) + } + if names, _ = q.ReadBuiltinNames(); !names["#only"] { + t.Fatal("Prune must keep the builtin names file") + } +} + +func TestBuiltinNamesBounded(t *testing.T) { + q := NewQueue(filepath.Join(t.TempDir(), QueueDirName)) + many := make([]string, 0, MaxBuiltinNames+50) + for i := 0; i < MaxBuiltinNames+50; i++ { + many = append(many, "#n"+string(rune('a'+i%26))+NewID()[:6]) + } + if err := q.WriteBuiltinNames(many); err != nil { + t.Fatal(err) + } + names, err := q.ReadBuiltinNames() + if err != nil { + t.Fatal(err) + } + if len(names) > MaxBuiltinNames { + t.Fatalf("read %d names, cap is %d", len(names), MaxBuiltinNames) + } + // A corrupt file reads as an error, not as a partial set. + if err := os.WriteFile(filepath.Join(q.Dir(), BuiltinNamesFileName), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := q.ReadBuiltinNames(); err == nil { + t.Fatal("corrupt file must be an error") + } +} diff --git a/internal/channelregistry/sql.go b/internal/channelregistry/sql.go new file mode 100644 index 000000000..dc54a8ee3 --- /dev/null +++ b/internal/channelregistry/sql.go @@ -0,0 +1,116 @@ +package channelregistry + +import ( + "context" + "database/sql" + "strings" +) + +// TableName is the table the ingestor creates (internal/dbschema) and owns. +const TableName = "channel_proposals" + +// MaxListed caps every read. The table itself is bounded by MaxPending, +// MaxApproved and retention; this is a second guard against an operator +// lowering MaxApproved below what is already approved. +const MaxListed = 1024 + +// IsMissingTable reports whether err is SQLite's "no such table" for the +// proposals table. That is the normal state while a new server runs against +// a database an older ingestor has not migrated yet, so readers treat it as +// an empty table rather than an error. +func IsMissingTable(err error) bool { + return err != nil && strings.Contains(err.Error(), "no such table: "+TableName) +} + +func clampLimit(limit int) int { + if limit <= 0 || limit > MaxListed { + return MaxListed + } + return limit +} + +// ListApprovedNames returns the approved channel names, oldest approval +// first. A missing table yields an empty result. +func ListApprovedNames(ctx context.Context, db *sql.DB, limit int) ([]string, error) { + rows, err := db.QueryContext(ctx, + `SELECT name FROM channel_proposals WHERE status = 'approved' + ORDER BY reviewed_at, created_at, id LIMIT ?`, clampLimit(limit)) + if err != nil { + if IsMissingTable(err) { + return nil, nil + } + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} + +// ListProposals returns proposals, newest first, optionally filtered by +// status ("" for all). A missing table yields an empty result. +func ListProposals(ctx context.Context, db *sql.DB, status string, limit int) ([]Proposal, error) { + query := `SELECT id, name, status, created_at, reviewed_at FROM channel_proposals` + args := []interface{}{} + if status != "" { + query += ` WHERE status = ?` + args = append(args, status) + } + query += ` ORDER BY created_at DESC, id LIMIT ?` + args = append(args, clampLimit(limit)) + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + if IsMissingTable(err) { + return []Proposal{}, nil + } + return nil, err + } + defer rows.Close() + out := []Proposal{} + for rows.Next() { + p, err := scanProposal(rows) + if err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// GetProposal returns one proposal. found is false when it does not exist or +// the table is missing. +func GetProposal(ctx context.Context, db *sql.DB, id string) (p Proposal, found bool, err error) { + row := db.QueryRowContext(ctx, + `SELECT id, name, status, created_at, reviewed_at FROM channel_proposals WHERE id = ?`, id) + p, err = scanProposal(row) + if err == sql.ErrNoRows || IsMissingTable(err) { + return Proposal{}, false, nil + } + if err != nil { + return Proposal{}, false, err + } + return p, true, nil +} + +type scanner interface { + Scan(dest ...interface{}) error +} + +func scanProposal(s scanner) (Proposal, error) { + var p Proposal + var reviewed sql.NullInt64 + if err := s.Scan(&p.ID, &p.Name, &p.Status, &p.CreatedAt, &reviewed); err != nil { + return Proposal{}, err + } + if reviewed.Valid { + v := reviewed.Int64 + p.ReviewedAt = &v + } + return p, nil +} diff --git a/internal/channelregistry/types.go b/internal/channelregistry/types.go new file mode 100644 index 000000000..81473ba1a --- /dev/null +++ b/internal/channelregistry/types.go @@ -0,0 +1,80 @@ +package channelregistry + +// Proposal statuses, as stored in channel_proposals.status. +const ( + StatusPending = "pending" + StatusApproved = "approved" + StatusRejected = "rejected" + // StatusRevoked marks a previously approved channel an administrator has + // undone. The row is kept (not deleted) so the audit trail and retention + // rules apply the same way as for rejected rows; see PruneChannelProposals. + StatusRevoked = "revoked" +) + +// ValidStatus reports whether s is a stored proposal status. +func ValidStatus(s string) bool { + return s == StatusPending || s == StatusApproved || s == StatusRejected || s == StatusRevoked +} + +// Request states reported by GET /api/channel-proposals/requests/{id}. +// "queued" means the ingestor has not picked the request up yet; the other +// states are the outcome it wrote back. +const ( + RequestQueued = "queued" + RequestPending = StatusPending + RequestApproved = StatusApproved + RequestRejected = StatusRejected + RequestRevoked = StatusRevoked + RequestError = "error" +) + +// Command operations. +const ( + OpSubmit = "submit" + OpApprove = "approve" + OpReject = "reject" + // OpRevoke undoes a previous approval, moving an approved channel back to + // revoked. It reuses Command.ProposalID, same as OpApprove/OpReject. + OpRevoke = "revoke" +) + +// Proposal is one suggested channel. Timestamps are Unix epoch milliseconds. +type Proposal struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + CreatedAt int64 `json:"createdAt"` + ReviewedAt *int64 `json:"reviewedAt,omitempty"` +} + +// Command is what the server writes into the queue. For OpSubmit, Name is set +// and the new proposal gets RequestID as its id, which is what makes replaying +// a command after a crash idempotent. For OpApprove/OpReject, ProposalID is set. +type Command struct { + RequestID string `json:"requestId"` + Op string `json:"op"` + Name string `json:"name,omitempty"` + ProposalID string `json:"proposalId,omitempty"` + CreatedAt int64 `json:"createdAt"` +} + +// Result is what the ingestor writes back once a command has been applied. +type Result struct { + RequestID string `json:"requestId"` + Status string `json:"status"` + Proposal *Proposal `json:"proposal,omitempty"` + Error string `json:"error,omitempty"` + CompletedAt int64 `json:"completedAt"` +} + +// RequestStatus is the public view of a request: the API response body of +// GET /api/channel-proposals/requests/{id}. +type RequestStatus struct { + Status string `json:"status"` + Proposal *Proposal `json:"proposal,omitempty"` + Error string `json:"error,omitempty"` + // CompletedAt is when the ingestor wrote the result (Unix ms; 0 while + // queued). Not part of the API response: the server uses it to tell a + // result it has not seen yet from one it is merely asked about again. + CompletedAt int64 `json:"-"` +} diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index d845dd1ed..191bdf34e 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -120,6 +120,9 @@ func Apply(rw *sql.DB, logf Logger) error { if err := ensureTransmissionsLastSeenColumn(rw, logf); err != nil { return fmt.Errorf("ensure transmissions.last_seen: %w", err) } + if err := ensureChannelProposalsTable(rw, logf); err != nil { + return fmt.Errorf("ensure channel_proposals: %w", err) + } if err := ensureTransmissionsRouteMaskColumn(rw, logf); err != nil { return fmt.Errorf("ensure transmissions.route_mask: %w", err) } @@ -1206,3 +1209,85 @@ func ensureObserverNeighborMetricsTable(rw *sql.DB, logf Logger) error { logf("[dbschema] created observer_neighbor_metrics table") return nil } + +// ensureChannelProposalsTable creates the table behind publicly suggested +// hashtag channels (internal/channelregistry), and upgrades one created +// before 'revoked' existed (see upgradeChannelProposalsStatusCheck). The +// ingestor is its only writer. It is deliberately NOT part of AssertReady: a +// new server must keep working against a database an older ingestor has not +// migrated yet, and its readers (channelregistry.ListApprovedNames / +// ListProposals) treat the missing table as empty on every query instead of +// caching its absence. +// +// name is COLLATE BINARY because the firmware derives the channel key from +// the exact bytes of the name: "#Test" and "#test" are different channels. +func ensureChannelProposalsTable(rw *sql.DB, logf Logger) error { + if _, err := rw.Exec(channelProposalsDDL("channel_proposals")); err != nil { + return fmt.Errorf("create channel_proposals: %w", err) + } + if err := upgradeChannelProposalsStatusCheck(rw, logf); err != nil { + return err + } + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_channel_proposals_status ON channel_proposals(status, created_at)`); err != nil { + return fmt.Errorf("create idx_channel_proposals_status: %w", err) + } + return nil +} + +// channelProposalsDDL is the current channel_proposals definition under the +// given table name. status includes 'revoked' (an approval an admin undid). +func channelProposalsDDL(table string) string { + return `CREATE TABLE IF NOT EXISTS ` + table + ` ( + id TEXT PRIMARY KEY, + name TEXT COLLATE BINARY NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN ('pending','approved','rejected','revoked')), + created_at INTEGER NOT NULL, + reviewed_at INTEGER NULL + )` +} + +// upgradeChannelProposalsStatusCheck rebuilds a channel_proposals table +// created by commit 4868622e, the first shared-channel-proposals commit, +// which was pushed and can have run against test databases. Its status CHECK +// lacks 'revoked', CREATE TABLE IF NOT EXISTS does not change an existing +// table, and SQLite cannot alter a CHECK in place, so every revoke would fail +// with "CHECK constraint failed". The rebuild keeps every row and runs in one +// transaction: a crash leaves either the old table or the new one, never +// neither. A table whose DDL already allows 'revoked' is left alone. +func upgradeChannelProposalsStatusCheck(rw *sql.DB, logf Logger) error { + var ddl string + err := rw.QueryRow(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'channel_proposals'`).Scan(&ddl) + if err != nil { + return fmt.Errorf("read channel_proposals DDL: %w", err) + } + if strings.Contains(ddl, "'revoked'") { + return nil + } + tx, err := rw.Begin() + if err != nil { + return fmt.Errorf("begin channel_proposals rebuild: %w", err) + } + defer tx.Rollback() + steps := []string{ + `DROP TABLE IF EXISTS channel_proposals_rebuild`, + channelProposalsDDL("channel_proposals_rebuild"), + `INSERT INTO channel_proposals_rebuild (id, name, status, created_at, reviewed_at) + SELECT id, name, status, created_at, reviewed_at FROM channel_proposals`, + `DROP TABLE channel_proposals`, + `ALTER TABLE channel_proposals_rebuild RENAME TO channel_proposals`, + } + for _, q := range steps { + if _, err := tx.Exec(q); err != nil { + return fmt.Errorf("rebuild channel_proposals: %w", err) + } + } + var n int + if err := tx.QueryRow(`SELECT COUNT(*) FROM channel_proposals`).Scan(&n); err != nil { + return fmt.Errorf("count rebuilt channel_proposals: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit channel_proposals rebuild: %w", err) + } + logf("[dbschema] rebuilt channel_proposals so status CHECK allows 'revoked' (%d row(s) kept)", n) + return nil +} diff --git a/internal/dbschema/dbschema_test.go b/internal/dbschema/dbschema_test.go index d920dca77..00100615d 100644 --- a/internal/dbschema/dbschema_test.go +++ b/internal/dbschema/dbschema_test.go @@ -2,6 +2,7 @@ package dbschema import ( "database/sql" + "fmt" "path/filepath" "strings" "testing" @@ -255,3 +256,142 @@ func TestPartialIdxTxLastSeenZero_FullIndexDropped(t *testing.T) { t.Fatalf("legacy idx_tx_last_seen must be dropped after partial index is in place (#1740); still present as %q", legacyName) } } + +// Shared channel proposals: Apply owns the table, names are unique with +// byte-exact (BINARY) semantics, and the server's AssertReady does NOT +// require the table, so a new server keeps starting against a database an +// older ingestor has not migrated yet (rolling upgrade). +func TestChannelProposalsTable(t *testing.T) { + db := minimalDB(t) + defer db.Close() + for i := 0; i < 2; i++ { // idempotent + if err := Apply(db, nil); err != nil { + t.Fatalf("Apply #%d: %v", i+1, err) + } + } + ins := func(id, name string) error { + _, err := db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at) VALUES (?, ?, 'pending', 1)`, id, name) + return err + } + if err := ins("a", "#Test"); err != nil { + t.Fatal(err) + } + if err := ins("b", "#test"); err != nil { + t.Fatalf("#test must be distinct from #Test: %v", err) + } + if err := ins("c", "#Test"); err == nil { + t.Fatal("duplicate #Test must violate UNIQUE") + } + if _, err := db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at) VALUES ('d', '#x', 'bogus', 1)`); err == nil { + t.Fatal("status CHECK not enforced") + } + if err := AssertReady(db); err != nil { + t.Fatalf("AssertReady after Apply: %v", err) + } + if _, err := db.Exec(`DROP TABLE channel_proposals`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err != nil { + t.Fatalf("AssertReady must not require channel_proposals: %v", err) + } +} + +// A database that ran commit 4868622e (the first shared-channel-proposals +// commit, deployed to test instances) has channel_proposals with a CHECK that +// does not allow 'revoked'. CREATE TABLE IF NOT EXISTS leaves that table +// alone, so Apply must rebuild it — keeping every row — or every revoke fails +// with "CHECK constraint failed". +func TestChannelProposalsTable_UpgradesCheckWithoutRevoked(t *testing.T) { + db := minimalDB(t) + defer db.Close() + if _, err := db.Exec(`CREATE TABLE channel_proposals ( + id TEXT PRIMARY KEY, + name TEXT COLLATE BINARY NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN ('pending','approved','rejected')), + created_at INTEGER NOT NULL, + reviewed_at INTEGER NULL + )`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE INDEX idx_channel_proposals_status ON channel_proposals(status, created_at)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at, reviewed_at) VALUES + ('a', '#Alpha', 'approved', 10, 20), + ('b', '#beta', 'pending', 11, NULL), + ('c', '#Gamma', 'rejected', 12, 22)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE channel_proposals SET status = 'revoked' WHERE id = 'a'`); err == nil { + t.Fatal("precondition: the old CHECK must reject 'revoked'") + } + + var logs []string + logf := func(format string, args ...interface{}) { logs = append(logs, fmt.Sprintf(format, args...)) } + if err := Apply(db, logf); err != nil { + t.Fatalf("Apply: %v", err) + } + rebuilt := 0 + for _, l := range logs { + if strings.Contains(l, "channel_proposals") && strings.Contains(l, "revoked") { + rebuilt++ + } + } + if rebuilt != 1 { + t.Fatalf("want exactly one log line about rebuilding channel_proposals for 'revoked', got %q", logs) + } + + rows, err := db.Query(`SELECT id, name, status, created_at, COALESCE(reviewed_at, -1) FROM channel_proposals ORDER BY id`) + if err != nil { + t.Fatal(err) + } + var got []string + for rows.Next() { + var id, name, status string + var created, reviewed int64 + if err := rows.Scan(&id, &name, &status, &created, &reviewed); err != nil { + t.Fatal(err) + } + got = append(got, fmt.Sprintf("%s|%s|%s|%d|%d", id, name, status, created, reviewed)) + } + rows.Close() + want := []string{"a|#Alpha|approved|10|20", "b|#beta|pending|11|-1", "c|#Gamma|rejected|12|22"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("rows after rebuild = %v, want %v", got, want) + } + if _, err := db.Exec(`UPDATE channel_proposals SET status = 'revoked' WHERE id = 'a'`); err != nil { + t.Fatalf("revoke after rebuild: %v", err) + } + if _, err := db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at) VALUES ('d', '#alpha', 'pending', 1)`); err != nil { + t.Fatalf("#alpha must still be distinct from #Alpha (BINARY): %v", err) + } + if _, err := db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at) VALUES ('e', '#Alpha', 'pending', 1)`); err == nil { + t.Fatal("UNIQUE(name) lost in rebuild") + } + if _, err := db.Exec(`INSERT INTO channel_proposals (id, name, status, created_at) VALUES ('f', '#x', 'bogus', 1)`); err == nil { + t.Fatal("status CHECK lost in rebuild") + } + var idx int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_channel_proposals_status'`).Scan(&idx); err != nil || idx != 1 { + t.Fatalf("status index after rebuild: n=%d err=%v", idx, err) + } + var leftovers int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'channel_proposals_%'`).Scan(&leftovers); err != nil || leftovers != 0 { + t.Fatalf("temporary table left behind: n=%d err=%v", leftovers, err) + } + + // A second run is a no-op: nothing is rebuilt or logged, rows unchanged. + logs = nil + if err := Apply(db, logf); err != nil { + t.Fatalf("second Apply: %v", err) + } + for _, l := range logs { + if strings.Contains(l, "channel_proposals") { + t.Fatalf("second Apply must not touch channel_proposals, logged %q", l) + } + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM channel_proposals`).Scan(&n); err != nil || n != 4 { + t.Fatalf("row count after second Apply = %d (%v), want 4", n, err) + } +} diff --git a/public/analytics.js b/public/analytics.js index 40c84bca6..e73d91403 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -1047,9 +1047,9 @@ var nameHtml = c.displayNameHtml ? c.displayNameHtml : esc(c.displayName || c.name || 'Unknown'); - return '' + + return '' + '' + nameHtml + '' + - '' + (typeof c.hash === 'number' ? '0x' + c.hash.toString(16).toUpperCase().padStart(2, '0') : c.hash) + '' + + '' + (typeof c.hash === 'number' ? '0x' + c.hash.toString(16).toUpperCase().padStart(2, '0') : esc(c.hash)) + '' + '' + c.messages + '' + '' + c.senders + '' + '' + timeAgo(c.lastActivity) + '' + diff --git a/public/app.js b/public/app.js index 6686b5f18..e6cd4e8dd 100644 --- a/public/app.js +++ b/public/app.js @@ -1999,7 +1999,7 @@ window.addEventListener('DOMContentLoaded', () => { const chList = Array.isArray(channels) ? channels : []; for (const c of chList) { if (c.name && c.name.toLowerCase().includes(q.toLowerCase())) { - html += `
+ html += `
Channel${escapeHtml(c.name)}
`; } } diff --git a/public/channel-proposals.css b/public/channel-proposals.css new file mode 100644 index 000000000..53d44cdc3 --- /dev/null +++ b/public/channel-proposals.css @@ -0,0 +1,100 @@ +/* Shared channel proposals (channel-proposals.js). Colors come only from the + * theme variables in style.css so the customizer and dark mode apply. */ + +/* Status line under the suggest form and in the admin dialog. */ +.ch-proposal-status { + font-size: 12px; + margin-top: 6px; + min-height: 1em; + color: var(--text-muted); +} +.ch-proposal-status[data-kind="success"] { color: var(--status-green); } +.ch-proposal-status[data-kind="warn"] { color: var(--status-yellow); } +.ch-proposal-status[data-kind="error"] { color: var(--status-red); } + +.ch-proposal-admin-link { + display: inline-block; + margin-top: 8px; + font-size: 12px; + color: var(--link-color, var(--accent)); +} + +/* "Shared" label on approved channels in the sidebar. */ +.ch-shared-badge { + flex-shrink: 0; + margin-left: 4px; + padding: 0 6px; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + font-size: 11px; + line-height: 16px; + color: var(--text-muted); +} + +/* Admin review dialog. */ +.ch-proposals-modal { max-width: 560px; } +.ch-proposals-label { + display: block; + margin-bottom: 6px; + font-size: 13px; + font-weight: 600; +} +.ch-proposals-toolbar { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: center; + margin-bottom: 10px; +} +.ch-proposals-filters { display: flex; flex-wrap: wrap; gap: 6px; margin-right: auto; } +.ch-proposals-filter[aria-pressed="true"] { + background: var(--accent); + border-color: var(--accent); + color: var(--text-on-accent); +} +.ch-proposals-list { + list-style: none; + margin: 0; + padding: 0; + max-height: min(60vh, 480px); + overflow-y: auto; +} +.ch-proposals-item { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: center; + justify-content: space-between; + padding: 10px 0; + border-top: 1px solid var(--border); +} +.ch-proposals-item:first-child { border-top: none; } +.ch-proposals-main { display: flex; flex-direction: column; min-width: 0; } +.ch-proposals-name { font-weight: 600; overflow-wrap: anywhere; } +.ch-proposals-meta { font-size: 12px; color: var(--text-muted); } +.ch-proposals-builtin { font-size: 12px; color: var(--warning); } +.ch-proposals-actions { display: flex; gap: 6px; align-items: center; } +.ch-proposals-actions button, +.ch-proposals-toolbar button { min-height: 44px; } +.ch-proposals-state { + font-size: 12px; + text-transform: capitalize; + color: var(--text-muted); +} +.ch-proposals-state[data-state="approved"] { color: var(--status-green); } +.ch-proposals-state[data-state="rejected"] { color: var(--status-red); } +.ch-proposals-empty { padding: 12px 0; font-size: 13px; color: var(--text-muted); } + +/* Remove (revoke) confirmation, nested inside the admin dialog's own + * overlay (see openConfirm in channel-proposals.js). Reuses .modal-overlay/ + * .modal/.ch-modal and .ch-modal-row so it looks identical to the rest of + * the channel modals; only the narrower max-width is specific to it. */ +.ch-proposals-confirm { max-width: 420px; } +.ch-proposals-confirm h4 { margin: 0 0 8px; font-size: 16px; } +.ch-proposals-confirm-actions { justify-content: flex-end; margin-top: 14px; margin-bottom: 0; } + +@media (max-width: 640px) { + .ch-proposals-actions { width: 100%; } + .ch-proposals-actions button { flex: 1; } + .ch-proposals-confirm-actions button { flex: 1; } +} diff --git a/public/channel-proposals.js b/public/channel-proposals.js new file mode 100644 index 000000000..79c7c4e31 --- /dev/null +++ b/public/channel-proposals.js @@ -0,0 +1,676 @@ +/* Shared channel proposals — suggest public hashtag channels for everyone, + * and let an administrator (the existing apiKey) approve or reject them. + * + * Server contract (cmd/server/channel_proposals.go): + * GET /api/channel-proposals/config → {enabled} + * POST /api/channel-proposals {name} → 202 {requestId} + * GET /api/channel-proposals/requests/{requestId} → {status, proposal, error} + * GET /api/admin/channel-proposals[?status=] → {proposals, enabled} (X-API-Key) + * POST /api/admin/channel-proposals/{id}/approve|reject → 202 {requestId} (X-API-Key) + * POST /api/admin/channel-proposals/{id}/revoke → 202 {requestId} or 409 (not approved, nothing queued) (X-API-Key) + * GET /api/channels → approvedChannels: [{name, hash}] + * + * The admin key lives only in this module's memory (never localStorage, + * sessionStorage or the URL) until "Lock" or a page reload. + * + * Pure helpers (normalizeName, mergeApprovedChannels, pollDelay, + * createPoller) have no DOM or network access so test-channel-proposals.js + * can run them in Node. + */ +(function (root) { + 'use strict'; + + // Firmware limit: ChannelDetails.name is char[32] including the NUL, so a + // name is at most 31 bytes of UTF-8 including the '#'. Mirrors + // internal/channelregistry.NormalizeName; the server stays authoritative. + var MAX_NAME_BYTES = 31; + var CONTROL_RE = /[\u0000-\u001F\u007F-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069\u2028\u2029]/; + var LEADING_SPACE_RE = /^\s/; + // Invisible formatting characters (Unicode category Cf) except the + // ZERO WIDTH JOINER emoji need; variation selectors are not Cf. CoreScope's + // own presentation rule, the same as isInvisibleFormat in + // internal/channelregistry (the firmware only limits the length). An + // engine without \p{} skips this check; the server still enforces it. + var INVISIBLE_RE = (function () { + try { return new RegExp('(?!\\u200D)\\p{Cf}', 'u'); } catch (e) { return null; } + })(); + + function utf8Length(s) { + // encodeURIComponent throws on lone surrogates, which are not valid UTF-8. + return encodeURIComponent(s).replace(/%[0-9A-F]{2}/gi, 'x').length; + } + + function normalizeName(raw) { + var s = String(raw == null ? '' : raw).trim(); + if (s.charAt(0) !== '#') s = '#' + s; + var body = s.slice(1); + if (!body) return { error: 'Enter a channel name.' }; + var bytes; + try { bytes = utf8Length(s); } catch (e) { return { error: 'The channel name contains invalid characters.' }; } + if (bytes > MAX_NAME_BYTES) return { error: 'Channel names can be at most ' + MAX_NAME_BYTES + ' bytes including the #.' }; + if (CONTROL_RE.test(s)) return { error: 'The channel name contains control characters.' }; + if (INVISIBLE_RE && INVISIBLE_RE.test(s)) return { error: 'The channel name contains invisible formatting characters.' }; + if (LEADING_SPACE_RE.test(body)) return { error: 'The channel name must not start with a space.' }; + return { name: s }; + } + + // Merge the server's approved shared channels into the channel list in + // O(n + m). Inputs are never mutated: channels that are also shared are + // shallow-copied with `shared: true`, and approved channels without traffic + // become new zero-count entries. + function mergeApprovedChannels(channels, approved) { + var list = Array.isArray(channels) ? channels : []; + if (!Array.isArray(approved) || approved.length === 0) return list.slice(); + var byHash = new Map(); + for (var i = 0; i < approved.length; i++) { + var a = approved[i]; + if (a && typeof a.hash === 'string' && a.hash && !byHash.has(a.hash)) byHash.set(a.hash, a); + } + var matched = new Set(); + var out = new Array(list.length); + for (var j = 0; j < list.length; j++) { + var ch = list[j]; + var hit = ch ? (byHash.get(ch.hash) || (ch.name ? byHash.get(ch.name) : undefined)) : undefined; + if (hit) { + out[j] = Object.assign({}, ch, { shared: true }); + matched.add(hit.hash); + } else { + out[j] = ch; + } + } + byHash.forEach(function (a, hash) { + if (matched.has(hash)) return; + out.push({ + hash: hash, + name: typeof a.name === 'string' && a.name ? a.name : hash, + messageCount: 0, + lastActivity: null, + lastActivityMs: 0, + lastMessage: null, + lastSender: null, + shared: true + }); + }); + return out; + } + + // Bounded backoff for polling one request: 1s, 1.5s, 2.25s … capped at 10s. + var MAX_POLL_ATTEMPTS = 20; // ≈ 2.5 minutes in total + function pollDelay(attempt) { + return Math.min(Math.round(1000 * Math.pow(1.5, Math.max(0, attempt))), 10000); + } + + // One poller per slot: starting a new request cancels the previous one. + // opts.fetchStatus(id) → Promise, opts.onUpdate(status), + // opts.onGiveUp(); timers are injectable for tests. + function createPoller(opts) { + var setT = opts.setTimeout || root.setTimeout.bind(root); + var clearT = opts.clearTimeout || root.clearTimeout.bind(root); + var maxAttempts = opts.maxAttempts || MAX_POLL_ATTEMPTS; + var timer = null; + var generation = 0; + + function cancel() { + generation++; + if (timer !== null) { clearT(timer); timer = null; } + } + + function start(requestId) { + cancel(); + var gen = generation; + var attempt = 0; + function schedule() { + if (attempt >= maxAttempts) { + timer = null; + if (opts.onGiveUp) opts.onGiveUp(); + return; + } + timer = setT(tick, pollDelay(attempt)); + attempt++; + } + function tick() { + timer = null; + Promise.resolve().then(function () { return opts.fetchStatus(requestId); }).then(function (st) { + if (gen !== generation) return; + opts.onUpdate(st); + if (st && st.status === 'queued') schedule(); + }, function (err) { + if (gen !== generation) return; + if (err && err.status === 404) { + opts.onUpdate({ status: 'error', error: 'The request expired. Please try again.' }); + return; + } + schedule(); // transient network/server error: keep backing off + }); + } + schedule(); + } + + return { start: start, cancel: cancel, isActive: function () { return timer !== null; } }; + } + + // ── Network ──────────────────────────────────────────────────────────── + var adminKey = null; // memory only + + function request(method, path, opts) { + opts = opts || {}; + var headers = { 'Accept': 'application/json' }; + if (opts.body !== undefined) headers['Content-Type'] = 'application/json'; + if (opts.admin && adminKey) headers['X-API-Key'] = adminKey; + return fetch('/api' + path, { + method: method, + headers: headers, + body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, + cache: 'no-store', + signal: opts.signal + }).then(function (res) { + return res.text().then(function (text) { + var data = null; + try { data = text ? JSON.parse(text) : null; } catch (e) { data = null; } + if (!res.ok) { + var err = new Error((data && data.error) || ('Request failed (' + res.status + ')')); + err.status = res.status; + throw err; + } + return data; + }); + }); + } + + function statusOf(requestId) { + return request('GET', '/channel-proposals/requests/' + encodeURIComponent(requestId)); + } + + // ── Shared state for the mounted page ───────────────────────────────── + var state = null; // { root, onApproved, suggestPoller, adminPoller, cleanups: [] } + + function esc(s) { + return String(s == null ? '' : s) + .replace(/&/g, '&').replace(//g, '>') + .replace(/"/g, '"').replace(/'/g, '''); + } + + function fmtTime(ms) { + if (!ms) return ''; + try { return new Date(ms).toLocaleString(); } catch (e) { return ''; } + } + + function listen(el, type, fn) { + el.addEventListener(type, fn); + if (state) state.cleanups.push(function () { el.removeEventListener(type, fn); }); + } + + // ── Suggest form (a section of the Add Channel modal) ───────────────── + var BUILTIN_NOTE = 'This site already decrypts it through its built-in channel list, so sharing it changes nothing.'; + + // Plain text for the suggest form's status line (shown with textContent). + function suggestMessage(st, fallbackName) { + var p = st && st.proposal; + var name = (p && p.name) || fallbackName || 'The channel'; + var note = st && st.builtIn ? ' ' + BUILTIN_NOTE : ''; + switch (st && st.status) { + case 'queued': return { text: 'Sending your suggestion…', kind: 'info' }; + case 'pending': return { text: 'Thanks! ' + name + ' is waiting for an administrator to review it.' + note, kind: note ? 'warn' : 'success' }; + case 'approved': return { text: name + ' is already shared with everyone.' + note, kind: 'success' }; + case 'rejected': return { text: name + ' was not accepted as a shared channel. A rejected name can be suggested again once the earlier decision expires (30 days by default).', kind: 'warn' }; + default: return { text: (st && st.error) || 'The suggestion could not be processed.', kind: 'error' }; + } + } + + function renderSuggestSection(section) { + section.innerHTML = + '

Suggest a Shared Channel

' + + '

Propose a public hashtag channel for everyone on this site. ' + + 'An administrator reviews each suggestion before it is shared. Only public hashtag channels can be suggested — never a private key.

' + + '
' + + '' + + '' + + '' + + '
' + + '
Case-sensitive, up to ' + MAX_NAME_BYTES + ' bytes including the #.
' + + '
' + + 'Review suggestions (admin)'; + section.hidden = false; + + var form = section.querySelector('#chSuggestForm'); + var input = section.querySelector('#chSuggestName'); + var btn = section.querySelector('#chSuggestBtn'); + var status = section.querySelector('#chSuggestStatus'); + function show(msg) { + status.textContent = msg.text; + status.setAttribute('data-kind', msg.kind); + } + listen(form, 'submit', function (e) { + e.preventDefault(); + var n = normalizeName(input.value); + if (n.error) { show({ text: n.error, kind: 'error' }); input.focus(); return; } + btn.disabled = true; + show({ text: 'Sending your suggestion…', kind: 'info' }); + request('POST', '/channel-proposals', { body: { name: n.name } }).then(function (res) { + if (!state) return; + state.suggestPoller.start(res.requestId); + state.suggestName = n.name; + input.value = ''; + }, function (err) { + if (err.status === 429) show({ text: 'Too many suggestions right now. Please try again later.', kind: 'error' }); + else if (err.status === 403) show({ text: 'Channel suggestions are closed.', kind: 'error' }); + else show({ text: err.message, kind: 'error' }); + }).then(function () { btn.disabled = false; }); + }); + state.suggestShow = show; + } + + // ── Admin dialog (#/channels?view=proposals) ─────────────────────────── + var FILTERS = [['pending', 'Pending'], ['approved', 'Approved'], ['rejected', 'Rejected'], ['revoked', 'Revoked']]; + + function adminEl() { return document.getElementById('chProposalsAdmin'); } + function confirmEl() { return document.getElementById('chProposalsConfirm'); } + + function focusables(container) { + return Array.prototype.filter.call( + container.querySelectorAll('button, [href], input, [tabindex]:not([tabindex="-1"])'), + function (el) { return !el.disabled && el.offsetParent !== null; }); + } + + function openAdmin() { + if (!state || adminEl()) return; + state.adminTrigger = document.activeElement; + state.adminFilter = state.adminFilter || 'pending'; + var overlay = document.createElement('div'); + overlay.id = 'chProposalsAdmin'; + overlay.className = 'modal-overlay ch-modal-overlay ch-proposals-overlay'; + overlay.setAttribute('role', 'dialog'); + overlay.setAttribute('aria-modal', 'true'); + overlay.setAttribute('aria-labelledby', 'chProposalsTitle'); + overlay.innerHTML = + '
' + + '' + + '

Channel Suggestions

' + + '
' + + '
' + + '
'; + state.root.appendChild(overlay); + overlay.addEventListener('click', onAdminClick); + overlay.addEventListener('submit', onAdminSubmit); + document.addEventListener('keydown', onAdminKeydown); + renderAdminBody(); + } + + function closeAdmin(opts) { + var el = adminEl(); + if (!el) return; + closeConfirm({ skipFocusReturn: true }); // never leave an orphaned confirm layer behind + if (state) state.adminPoller.cancel(); + document.removeEventListener('keydown', onAdminKeydown); + el.remove(); + // Leave the deep link without re-running the page's init. + if (!(opts && opts.keepHash) && /[?&]view=proposals\b/.test(location.hash)) { + history.replaceState(null, '', '#/channels'); + } + var back = state && state.adminTrigger; + if (back && back !== document.body && document.contains(back) && typeof back.focus === 'function') back.focus(); + else { + var addBtn = document.getElementById('chAddChannelBtn'); + if (addBtn) addBtn.focus(); + } + } + + function adminStatus(text, kind) { + var s = document.getElementById('chProposalsStatus'); + if (!s) return; + s.textContent = text || ''; + s.setAttribute('data-kind', kind || 'info'); + } + + function renderAdminBody() { + var body = document.getElementById('chProposalsBody'); + if (!body) return; + if (!adminKey) { + body.innerHTML = + '
' + + '' + + '
' + + '' + + '' + + '
' + + '
The key is kept in this tab\'s memory only and is forgotten on reload.
' + + '
'; + var k = document.getElementById('chProposalsKey'); + if (k) k.focus(); + return; + } + body.innerHTML = + '
' + + '
' + + FILTERS.map(function (f) { + var on = state.adminFilter === f[0]; + return ''; + }).join('') + + '
' + + '' + + '' + + '
' + + '
    '; + var active = body.querySelector('[aria-pressed="true"]'); + if (active) active.focus(); + loadAdminList(); + } + + function loadAdminList() { + var list = document.getElementById('chProposalsList'); + if (!list) return; + list.setAttribute('aria-busy', 'true'); + var filter = state.adminFilter; + request('GET', '/admin/channel-proposals?status=' + encodeURIComponent(filter), { admin: true }).then(function (data) { + if (!state || state.adminFilter !== filter || !document.getElementById('chProposalsList')) return; + var items = (data && Array.isArray(data.proposals)) ? data.proposals : []; + list.innerHTML = items.length ? items.map(renderAdminRow).join('') : + '
  • No ' + esc(filter) + ' suggestions.
    • '; + list.setAttribute('aria-busy', 'false'); + if (data && data.enabled === false && filter === 'pending') { + adminStatus('Public suggestions are currently closed; existing ones can still be reviewed.', 'info'); + } + }, onAdminError); + } + + function renderAdminRow(p) { + var when = p.status === 'pending' + ? 'Suggested ' + esc(fmtTime(p.createdAt)) + : 'Reviewed ' + esc(fmtTime(p.reviewedAt || p.createdAt)); + var actions; + if (p.status === 'pending') { + actions = '
    ' + + '' + + '' + + '
    '; + } else if (p.status === 'approved') { + // Approved rows additionally get a Remove (revoke) action, guarded by + // a confirmation step (see openConfirm) before anything is sent. + actions = '
    ' + + 'approved' + + '' + + '
    '; + } else { + actions = '' + esc(p.status) + ''; + } + // A name in the ingestor's built-in list (rainbow table / config) + // decrypts whatever the decision: say so before anyone approves it or + // expects a Remove to stop it. + var builtin = p.builtIn === true + ? 'Built in: already decrypted' + : ''; + return '
  • ' + + '
    ' + esc(p.name) + '' + builtin + + '' + when + '
    ' + actions + '
    • '; + } + + // ── Remove (revoke) confirmation layer, nested inside the admin dialog ── + // Reuses the admin dialog's own overlay/modal classes and its single + // document keydown listener (onAdminKeydown), which checks confirmEl() + // first so Escape/Tab-trapping apply to whichever layer is on top — + // never two independent keydown listeners fighting over the same keys. + function makeEl(tag, className, text) { + var e = document.createElement(tag); + if (className) e.className = className; + if (text != null) e.textContent = text; + return e; + } + + // Built with the DOM API: the channel name only ever reaches textContent + // and setAttribute, never an HTML string. + function openConfirm(id, name, builtIn) { + if (!state || !adminEl() || confirmEl()) return; + state.confirmTrigger = document.activeElement; + var dlg = document.createElement('div'); + dlg.id = 'chProposalsConfirm'; + dlg.className = 'modal-overlay ch-modal-overlay ch-proposals-confirm-overlay'; + dlg.setAttribute('role', 'alertdialog'); + dlg.setAttribute('aria-modal', 'true'); + dlg.setAttribute('aria-labelledby', 'chProposalsConfirmTitle'); + + var modal = makeEl('div', 'modal ch-modal ch-proposals-confirm'); + modal.setAttribute('role', 'document'); + var title = makeEl('h4', null, 'Remove ' + name + '?'); + title.id = 'chProposalsConfirmTitle'; + var hint = makeEl('p', 'ch-modal-section-hint', builtIn + ? 'It will stop being listed as shared, but this site keeps decrypting it through its built-in channel list.' + : 'It will stop being shared with everyone.'); + var row = makeEl('div', 'ch-modal-row ch-proposals-confirm-actions'); + var cancelBtn = makeEl('button', 'ch-modal-btn-secondary', 'Cancel'); + cancelBtn.setAttribute('type', 'button'); + cancelBtn.setAttribute('data-proposals-confirm-action', 'cancel'); + var confirmBtn = makeEl('button', 'btn-primary', 'Remove'); + confirmBtn.setAttribute('type', 'button'); + confirmBtn.setAttribute('data-proposals-confirm-action', 'confirm'); + confirmBtn.setAttribute('data-proposal-id', id); + confirmBtn.setAttribute('data-proposal-name', name); + row.appendChild(cancelBtn); + row.appendChild(confirmBtn); + modal.appendChild(title); + modal.appendChild(hint); + modal.appendChild(row); + dlg.appendChild(modal); + + adminEl().appendChild(dlg); + confirmBtn.focus(); + } + + function closeConfirm(opts) { + var el = confirmEl(); + if (!el) return; + el.remove(); + if (opts && opts.skipFocusReturn) return; + var back = state && state.confirmTrigger; + if (back && document.contains(back) && typeof back.focus === 'function') back.focus(); + } + + function onAdminError(err) { + if (err && (err.status === 401 || err.status === 403)) { + adminKey = null; + renderAdminBody(); + adminStatus(err.status === 401 ? 'The API key was not accepted.' : err.message, 'error'); + return; + } + adminStatus((err && err.message) || 'Something went wrong.', 'error'); + } + + function decide(id, op, btn) { + var row = btn.closest('.ch-proposals-item'); + var name = row ? (row.querySelector('.ch-proposals-name') || {}).textContent : ''; + if (row) Array.prototype.forEach.call(row.querySelectorAll('button'), function (b) { b.disabled = true; }); + var verb = op === 'approve' ? 'Approving ' : op === 'revoke' ? 'Removing ' : 'Rejecting '; + adminStatus(verb + name + '…', 'info'); + request('POST', '/admin/channel-proposals/' + encodeURIComponent(id) + '/' + op, { admin: true }).then(function (res) { + if (!state) return; + state.adminPending = { op: op, name: name }; + state.adminPoller.start(res.requestId); + }, function (err) { + if (row) Array.prototype.forEach.call(row.querySelectorAll('button'), function (b) { b.disabled = false; }); + onAdminError(err); + }); + } + + function onAdminDecision(st) { + if (!state) return; + if (st.status === 'queued') return; + var name = (st.proposal && st.proposal.name) || (state.adminPending && state.adminPending.name) || 'The channel'; + if (st.status === 'approved') { + adminStatus(name + ' is now shared with everyone.', 'success'); + if (typeof state.onApproved === 'function') state.onApproved(name); + } else if (st.status === 'rejected') { + adminStatus(name + ' was rejected.', 'success'); + } else if (st.status === 'revoked') { + adminStatus(name + ' was removed and is no longer shared.', 'success'); + } else { + adminStatus(st.error || 'The decision could not be applied.', 'error'); + } + loadAdminList(); + } + + function onAdminClick(e) { + var overlay = adminEl(); + var t = e.target; + // The confirm layer sits on top when open: its own backdrop click and + // its Cancel/Confirm buttons are handled here first, before anything + // that would act on the admin dialog underneath it. + if (confirmEl()) { + if (t === confirmEl()) { closeConfirm(); return; } + var ca = t.closest && t.closest('[data-proposals-confirm-action]'); + if (ca) { + var action = ca.getAttribute('data-proposals-confirm-action'); + if (action === 'cancel') { closeConfirm(); return; } + if (action === 'confirm') { + var confirmId = ca.getAttribute('data-proposal-id'); + var trigger = state && state.confirmTrigger; + closeConfirm({ skipFocusReturn: true }); + if (trigger) decide(confirmId, 'revoke', trigger); + return; + } + } + return; // swallow everything else while the confirm layer is open + } + if (t === overlay || (t.closest && t.closest('[data-proposals-action="close"]'))) { + e.preventDefault(); + closeAdmin(); + return; + } + var f = t.closest && t.closest('[data-proposals-filter]'); + if (f) { + state.adminFilter = f.getAttribute('data-proposals-filter'); + Array.prototype.forEach.call(overlay.querySelectorAll('[data-proposals-filter]'), function (b) { + b.setAttribute('aria-pressed', b === f ? 'true' : 'false'); + }); + adminStatus(''); + loadAdminList(); + return; + } + var a = t.closest && t.closest('[data-proposals-action]'); + if (a && a.getAttribute('data-proposals-action') === 'refresh') { loadAdminList(); return; } + if (a && a.getAttribute('data-proposals-action') === 'lock') { + adminKey = null; + state.adminPoller.cancel(); + renderAdminBody(); + adminStatus('Locked. The key was forgotten.', 'info'); + return; + } + var d = t.closest && t.closest('[data-proposals-decide]'); + if (d && !d.disabled) { + var op = d.getAttribute('data-proposals-decide'); + if (op === 'revoke') { + openConfirm(d.getAttribute('data-proposal-id'), d.getAttribute('data-proposal-name') || '', + d.getAttribute('data-proposal-builtin') === 'true'); + return; + } + decide(d.getAttribute('data-proposal-id'), op, d); + } + } + + function onAdminSubmit(e) { + if (e.target.id !== 'chProposalsKeyForm') return; + e.preventDefault(); + var input = document.getElementById('chProposalsKey'); + var key = input ? input.value.trim() : ''; + if (!key) { adminStatus('Enter the admin API key.', 'error'); return; } + adminKey = key; + if (input) input.value = ''; + adminStatus(''); + renderAdminBody(); + } + + // Single document-level keydown listener for the whole admin dialog, + // including the confirm layer nested inside it: whichever is currently on + // top handles Escape and Tab-trapping, so there is never a second, + // independently-stacked listener fighting this one over the same keys. + function onAdminKeydown(e) { + var overlay = adminEl(); + if (!overlay) return; + var layer = confirmEl() || overlay; + if (e.key === 'Escape') { + e.preventDefault(); + e.stopPropagation(); + if (layer === overlay) closeAdmin(); else closeConfirm(); + return; + } + if (e.key !== 'Tab') return; + var f = focusables(layer); + if (!f.length) return; + var first = f[0]; + var last = f[f.length - 1]; + if (!layer.contains(document.activeElement)) { e.preventDefault(); first.focus(); return; } + if (e.shiftKey && document.activeElement === first) { e.preventDefault(); last.focus(); } + else if (!e.shiftKey && document.activeElement === last) { e.preventDefault(); first.focus(); } + } + + // ── Lifecycle ───────────────────────────────────────────────────────── + // mount({root, suggestSection, view, onApproved}) is called from the + // Channels page init; unmount() from its destroy, which also stops polling. + function mount(opts) { + unmount(); + state = { + root: opts.root, + onApproved: opts.onApproved, + cleanups: [], + adminFilter: 'pending' + }; + state.suggestPoller = createPoller({ + fetchStatus: statusOf, + onUpdate: function (st) { + if (state && state.suggestShow) state.suggestShow(suggestMessage(st, state.suggestName)); + }, + onGiveUp: function () { + if (state && state.suggestShow) state.suggestShow({ text: 'Your suggestion is still being processed. Check back later.', kind: 'info' }); + } + }); + state.adminPoller = createPoller({ + fetchStatus: statusOf, + onUpdate: onAdminDecision, + onGiveUp: function () { adminStatus('The decision is still being processed. Refresh in a moment.', 'info'); } + }); + var section = opts.suggestSection; + if (section) { + var mine = state; + request('GET', '/channel-proposals/config').then(function (cfg) { + if (state !== mine || !cfg || cfg.enabled !== true) return; + renderSuggestSection(section); + }, function () { /* feature unavailable: keep the section hidden */ }); + } + if (opts.view === 'proposals') openAdmin(); + } + + function unmount() { + if (!state) return; + state.suggestPoller.cancel(); + state.adminPoller.cancel(); + var el = adminEl(); + if (el) el.remove(); + document.removeEventListener('keydown', onAdminKeydown); + state.cleanups.forEach(function (fn) { fn(); }); + state = null; + } + + root.ChannelProposals = { + MAX_NAME_BYTES: MAX_NAME_BYTES, + MAX_POLL_ATTEMPTS: MAX_POLL_ATTEMPTS, + FILTERS: FILTERS, + normalizeName: normalizeName, + mergeApprovedChannels: mergeApprovedChannels, + pollDelay: pollDelay, + createPoller: createPoller, + renderAdminRow: renderAdminRow, + suggestMessage: suggestMessage, + mount: mount, + unmount: unmount, + openAdmin: openAdmin, + closeAdmin: closeAdmin + }; +})(typeof window !== 'undefined' ? window : this); diff --git a/public/channels.js b/public/channels.js index e5cc11448..dd12fd335 100644 --- a/public/channels.js +++ b/public/channels.js @@ -815,6 +815,10 @@
    Case-sensitive — #meshcore ≠ #MeshCore
    + + +
    Keys stay in your browser — CoreScope is a passive observer that monitors and decrypts traffic but cannot transmit over RF. Use the close button to remove individual channels.
    @@ -892,6 +896,23 @@ } var addBtn = document.getElementById('chAddChannelBtn'); if (addBtn) addBtn.addEventListener('click', openAddModal); + + // Shared channel proposals: the suggest form lives in the Add modal and + // #/channels?view=proposals opens the admin review dialog. + if (window.ChannelProposals) { + window.ChannelProposals.mount({ + root: app, + suggestSection: document.getElementById('chSuggestSection'), + view: _initUrlParams.get('view'), + onApproved: function () { + invalidateApiCache('/channels'); + loadChannels(true).then(function () { + mergeUserChannels(); + renderChannelList(); + }); + } + }); + } if (modalEl) { modalEl.addEventListener('click', function (e) { // Close on overlay backdrop click or any [data-action=ch-modal-close] @@ -1646,24 +1667,31 @@ } // Tick relative timestamps every 1s — iterates channels array, updates DOM text only - timeAgoTimer = setInterval(function () { - var now = Date.now(); - for (var i = 0; i < channels.length; i++) { - var ch = channels[i]; - if (!ch.lastActivityMs) continue; - var text = formatSecondsAgo(Math.floor((now - ch.lastActivityMs) / 1000)); - var el = document.querySelector('.ch-item-time[data-channel-hash="' + ch.hash + '"]'); - if (el) el.textContent = text; - // #1367: mobile rows live in a flat list; update those too. - var rowEl = document.querySelector('.ch-row[data-hash="' + ch.hash + '"] .ch-row-time'); - if (rowEl) rowEl.textContent = text; - } - }, 1000); + timeAgoTimer = setInterval(function () { tickChannelTimes(channels, Date.now()); }, 1000); + } + + // Updates the relative "last activity" text of every rendered channel row. + // ch.hash is the channel name for hashtag channels, and shared channel + // names may contain quotes or backslashes, so it is CSS.escape'd before it + // goes into a selector (unescaped, a '"' throws and stops the loop). + function tickChannelTimes(list, now) { + for (var i = 0; i < list.length; i++) { + var ch = list[i]; + if (!ch.lastActivityMs) continue; + var text = formatSecondsAgo(Math.floor((now - ch.lastActivityMs) / 1000)); + var sel = CSS.escape(String(ch.hash)); + var el = document.querySelector('.ch-item-time[data-channel-hash="' + sel + '"]'); + if (el) el.textContent = text; + // #1367: mobile rows live in a flat list; update those too. + var rowEl = document.querySelector('.ch-row[data-hash="' + sel + '"] .ch-row-time'); + if (rowEl) rowEl.textContent = text; + } } var timeAgoTimer = null; function destroy() { + if (window.ChannelProposals) window.ChannelProposals.unmount(); if (wsHandler) offWS(wsHandler); wsHandler = null; if (timeAgoTimer) clearInterval(timeAgoTimer); @@ -1691,7 +1719,13 @@ channels = (data.channels || []).map(ch => { ch.lastActivityMs = ch.lastActivity ? new Date(ch.lastActivity).getTime() : 0; return ch; - }).sort((a, b) => (b.lastActivityMs || 0) - (a.lastActivityMs || 0)); + }); + // Approved shared channels are listed for everyone, even before they + // carry traffic. + if (window.ChannelProposals) { + channels = window.ChannelProposals.mergeApprovedChannels(channels, data.approvedChannels); + } + channels.sort((a, b) => (b.lastActivityMs || 0) - (a.lastActivityMs || 0)); renderChannelList(); reconcileSelectionAfterChannelRefresh(); } catch (e) { @@ -1729,6 +1763,10 @@ function renderChannelRow(ch) { const isEncrypted = ch.encrypted === true; const isUserAdded = ch.userAdded === true; + // Shared (approved) channels belong to everyone: no local remove/share + // controls, even when this browser also saved a key for the same name. + const isShared = ch.shared === true; + const managesLocalKey = isUserAdded && !isShared; // #1041: route through channelDisplayName so the psk:* → "Private // Channel" rule lives in one place. Pass an `encryptedFallback` so // rows for non-user-added encrypted channels keep showing "Unknown" @@ -1748,11 +1786,13 @@ preview = `0x${formatHashHex(ch.hash)}`; } else if (typeof ch.messageCount === 'number' && ch.messageCount > 0) { preview = `${ch.messageCount} messages`; + } else if (isShared) { + preview = 'Shared channel · no messages yet'; } else { preview = ''; } const sel = selectedHash === ch.hash ? ' selected' : ''; - const encClass = isUserAdded + const encClass = managesLocalKey ? ' ch-user-added' : (isEncrypted ? ' ch-encrypted' : ''); const badgeIcon = isUserAdded ? '' : (isEncrypted ? '' : null); @@ -1774,26 +1814,27 @@ + ' title="' + title + '"' + ' aria-label="' + ariaVerb + ' ' + escapeHtml(name) + '">' + glyph + ''; } - const removeBtn = isUserAdded + const removeBtn = managesLocalKey ? iconBtn('ch-remove-btn', 'data-remove-channel', ch.hash, name, '', 'Remove channel and clear saved key', 'Remove', '') : ''; - const shareBtn = isUserAdded + const shareBtn = managesLocalKey ? iconBtn('ch-share-btn', 'data-share-channel', ch.hash, name, ' Share', 'Share channel key (QR + URL)', 'Share', ' aria-haspopup="dialog"') : ''; const userBadge = isUserAdded ? ' ' : ''; + const sharedBadge = isShared ? ' Shared' : ''; const unreadBadge = (ch.unread && ch.unread > 0) ? ' ' + (ch.unread > 99 ? '99+' : ch.unread) + '' : ''; - return `