From 40eb9386752fbbe8605f903206991dd6c5e4a5e7 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:23:58 +0600 Subject: [PATCH 01/16] chore(ignore): ignore memgit/ standalone project memgit/ is a separate project, not part of membuss (like memploy/). Co-Authored-By: Claude --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 777d2ec..adb8737 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ Screenshot 2026-07-17 105706.png records.txt finding.txt -# Standalone Memploy project +# Standalone projects /memploy/ membussmobo +/memgit/ From 3c4637bb07bd17fc5bd70bdee32fa94e2825aae1 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:24:05 +0600 Subject: [PATCH 02/16] chore(lint): add golangci-lint v2 config (XC-004) standard preset, generated-code exclusions, test-file errcheck exemption. CI lint job runs reporting-only until debt burn-down. Co-Authored-By: Claude --- .golangci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..df77d6b --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,23 @@ +version: "2" + +run: + timeout: 10m + +linters: + # Standard set: errcheck, govet, ineffassign, staticcheck, default: standard + default: standard + settings: + staticcheck: + checks: + - all + # ST1000 package-comment + ST1003 naming too disruptive for existing codebase. + - -ST1000 + - -ST1003 + exclusions: + # Skip files carrying "// Code generated ... DO NOT EDIT." header (rpc/proto/*.pb.go). + generated: lax + rules: + # Tests routinely ignore returned errors on purpose. + - path: _test\.go + linters: + - errcheck From 9b249e4db613420689cec5ed96a7151ff12f62cf Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:24:21 +0600 Subject: [PATCH 03/16] test(fuzz): wire-protocol fuzz targets for all parsers (XC-001) Six Go-native fuzz targets over the external input surface: PEX frames, memex v2 frames, descriptors, MemNS record validation, keyring PEM import, HTTP Range parsing. Shared libp2p network.Stream test double in internal/wiretest. Fuzzing found a real bug on first run (GATE-011): parseRange("bytes=-1", size=0) returned success with an empty range. Fixed with an end<=start guard; counterexample preserved in gateway/memgate_v2/testdata/fuzz/FuzzParseRange/. Co-Authored-By: Claude --- core/descriptor/fuzz_test.go | 42 ++++++++++++++++++ core/keyring/fuzz_test.go | 29 +++++++++++++ gateway/memgate_v2/fuzz_test.go | 28 ++++++++++++ .../fuzz/FuzzParseRange/d036aa483bfab76b | 3 ++ internal/wiretest/stream.go | 43 +++++++++++++++++++ net/dht/fuzz_test.go | 24 +++++++++++ net/memex_v2/fuzz_test.go | 32 ++++++++++++++ net/pex/fuzz_test.go | 30 +++++++++++++ 8 files changed, 231 insertions(+) create mode 100644 core/descriptor/fuzz_test.go create mode 100644 core/keyring/fuzz_test.go create mode 100644 gateway/memgate_v2/fuzz_test.go create mode 100644 gateway/memgate_v2/testdata/fuzz/FuzzParseRange/d036aa483bfab76b create mode 100644 internal/wiretest/stream.go create mode 100644 net/dht/fuzz_test.go create mode 100644 net/memex_v2/fuzz_test.go create mode 100644 net/pex/fuzz_test.go diff --git a/core/descriptor/fuzz_test.go b/core/descriptor/fuzz_test.go new file mode 100644 index 0000000..244f614 --- /dev/null +++ b/core/descriptor/fuzz_test.go @@ -0,0 +1,42 @@ +// Fuzz target for descriptor parsing (finding.txt XC-001). +// Parse consumes untrusted .mbuss bytes from the network; magic, +// version, checksum, protobuf payload and embedded MIDs are all +// attacker-controlled. +package descriptor + +import ( + "testing" + + "github.com/nnlgsakib/membuss/core/mid" +) + +func FuzzDescriptorParse(f *testing.F) { + // Seed: a valid serialized descriptor built the same way the + // existing tests build one. + root := mid.FromBytes([]byte("fuzz-root")) + d := &Descriptor{ + RootMID: root, + TotalSize: 4, + Name: "fuzz", + MimeType: "application/octet-stream", + } + d.Blocks = []BlockEntry{{MID: mid.FromBytes([]byte("b1")), Size: 4}} + blob, err := d.Serialize() + if err == nil { + f.Add(blob) + } + f.Add([]byte{}) + f.Add([]byte("MEMB")) + f.Add([]byte("MEMB\x01")) + f.Add(make([]byte, 37)) + + f.Fuzz(func(t *testing.T, data []byte) { + dsc, err := Parse(data) + if err != nil { + return + } + if dsc.RootMID.IsZero() || len(dsc.Blocks) == 0 && dsc.TotalSize > 0 { + t.Fatalf("parsed descriptor with zero root MID or missing blocks: %+v", dsc) + } + }) +} diff --git a/core/keyring/fuzz_test.go b/core/keyring/fuzz_test.go new file mode 100644 index 0000000..811412c --- /dev/null +++ b/core/keyring/fuzz_test.go @@ -0,0 +1,29 @@ +// Fuzz target for keyring PEM import parsing (finding.txt XC-001). +// Import() runs attacker-controlled PEM bytes through pem.Decode +// and libp2p key unmarshalling before any disk IO; this target +// exercises exactly that parse prefix. +package keyring + +import ( + "testing" + + "encoding/pem" + "github.com/libp2p/go-libp2p/core/crypto" +) + +func FuzzKeyImportParse(f *testing.F) { + f.Add([]byte("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----\n")) + f.Add([]byte{0x00}) + f.Add([]byte("-----BEGIN CERTIFICATE-----\nZm9v\n-----END CERTIFICATE-----\n")) + + f.Fuzz(func(t *testing.T, data []byte) { + block, _ := pem.Decode(data) + if block == nil { + return + } + if block.Type != "PRIVATE KEY" { + return + } + _, _ = crypto.UnmarshalPrivateKey(block.Bytes) + }) +} diff --git a/gateway/memgate_v2/fuzz_test.go b/gateway/memgate_v2/fuzz_test.go new file mode 100644 index 0000000..b8a304a --- /dev/null +++ b/gateway/memgate_v2/fuzz_test.go @@ -0,0 +1,28 @@ +// Fuzz target for gateway Range header parsing (finding.txt XC-001). +// parseRange consumes attacker-controlled header strings on every +// media request; bounds math must stay inside [0,size]. +package memgate_v2 + +import ( + "testing" +) + +func FuzzParseRange(f *testing.F) { + f.Add("bytes=0-", int64(100)) + f.Add("bytes=-5", int64(100)) + f.Add("bytes=5-9", int64(100)) + f.Add("bytes=", int64(100)) + f.Add("", int64(0)) + f.Add("bytes=0-18446744073709551615", int64(100)) + f.Add("bytes=18446744073709551615-", int64(100)) + + f.Fuzz(func(t *testing.T, spec string, size int64) { + start, end, err := parseRange(spec, size) + if err != nil { + return + } + if start < 0 || end > size || start >= end { + t.Fatalf("parseRange(%q,%d) = [%d,%d): out of bounds", spec, size, start, end) + } + }) +} diff --git a/gateway/memgate_v2/testdata/fuzz/FuzzParseRange/d036aa483bfab76b b/gateway/memgate_v2/testdata/fuzz/FuzzParseRange/d036aa483bfab76b new file mode 100644 index 0000000..330435d --- /dev/null +++ b/gateway/memgate_v2/testdata/fuzz/FuzzParseRange/d036aa483bfab76b @@ -0,0 +1,3 @@ +go test fuzz v1 +string("bytes=-1") +int64(0) diff --git a/internal/wiretest/stream.go b/internal/wiretest/stream.go new file mode 100644 index 0000000..1070e02 --- /dev/null +++ b/internal/wiretest/stream.go @@ -0,0 +1,43 @@ +// Package wiretest provides in-memory test doubles for membuss +// wire-format helpers. Test-support code: never import from +// production paths. +package wiretest + +import ( + "io" + "time" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/protocol" +) + +// Stream is an inert network.Stream backed by a caller-supplied +// io.Reader and optional io.Writer. Only Read/Write carry behavior; +// every other method is a no-op — exactly the surface framing +// helpers (readMsg/readFrame) exercise. +// +// A nil Writer is replaced with io.Discard so writes never panic. +type Stream struct { + io.Reader + io.Writer +} + +// NewStream wraps r as a network.Stream whose reads drain r. +func NewStream(r io.Reader) *Stream { + return &Stream{Reader: r, Writer: io.Discard} +} + +func (s *Stream) ID() string { return "wiretest" } +func (s *Stream) Protocol() protocol.ID { return "/membuss/wiretest/1.0.0" } +func (s *Stream) SetProtocol(protocol.ID) error { return nil } +func (s *Stream) Stat() network.Stats { return network.Stats{} } +func (s *Stream) Conn() network.Conn { return nil } +func (s *Stream) Scope() network.StreamScope { return nil } +func (s *Stream) Close() error { return nil } +func (s *Stream) CloseRead() error { return nil } +func (s *Stream) CloseWrite() error { return nil } +func (s *Stream) Reset() error { return nil } +func (s *Stream) ResetWithError(network.StreamErrorCode) error { return nil } +func (s *Stream) SetDeadline(time.Time) error { return nil } +func (s *Stream) SetReadDeadline(time.Time) error { return nil } +func (s *Stream) SetWriteDeadline(time.Time) error { return nil } diff --git a/net/dht/fuzz_test.go b/net/dht/fuzz_test.go new file mode 100644 index 0000000..8f0147b --- /dev/null +++ b/net/dht/fuzz_test.go @@ -0,0 +1,24 @@ +// Fuzz target for MemNS record validation (finding.txt XC-001). +// validateMemNS unmarshals attacker-controlled protobuf records, +// verifies signatures, owner/delegate binding and name binding. +package dht + +import ( + "testing" + + membusspb "github.com/nnlgsakib/membuss/proto" + "google.golang.org/protobuf/proto" +) + +func FuzzMemNSValidate(f *testing.F) { + f.Add("/memns/k1", []byte{}) + var empty membusspb.MemNSRecord + if b, err := proto.Marshal(&empty); err == nil { + f.Add("/memns/k1", b) + } + f.Add("/memns/k1", []byte{0x0a, 0x03, 0x61, 0x62, 0x63}) + + f.Fuzz(func(t *testing.T, key string, value []byte) { + _ = validateMemNS(key, value) + }) +} diff --git a/net/memex_v2/fuzz_test.go b/net/memex_v2/fuzz_test.go new file mode 100644 index 0000000..fed3957 --- /dev/null +++ b/net/memex_v2/fuzz_test.go @@ -0,0 +1,32 @@ +// Fuzz targets for memex v2 wire format (finding.txt XC-001). +// readFrame is the entry point for all inbound protocol frames; +// after framing succeeds the payload goes through protobuf +// unmarshal, which this target also exercises. +package memex_v2 + +import ( + "bytes" + "testing" + + "github.com/nnlgsakib/membuss/internal/wiretest" + membusspb "github.com/nnlgsakib/membuss/proto" + "google.golang.org/protobuf/proto" +) + +func FuzzMemexReadFrame(f *testing.F) { + f.Add(append([]byte{0x00, 0x00, 0x10, 0x00}, make([]byte, 0x1000)...)) + f.Add([]byte{0x00, 0x00, 0x00, 0x01, 0x08}) + f.Add([]byte{}) + f.Add([]byte{0xFF}) + f.Add(append([]byte{0x7F, 0xFF, 0xFF, 0xFF}, make([]byte, 16)...)) + + f.Fuzz(func(t *testing.T, data []byte) { + s := wiretest.NewStream(bytes.NewReader(data)) + frame := readFrame(s) + if frame == nil { + return + } + var m membusspb.MemexMessage + _ = proto.Unmarshal(frame, &m) + }) +} diff --git a/net/pex/fuzz_test.go b/net/pex/fuzz_test.go new file mode 100644 index 0000000..7ad505c --- /dev/null +++ b/net/pex/fuzz_test.go @@ -0,0 +1,30 @@ +// Fuzz targets for PEX wire-format parsing (finding.txt XC-001). +// readMsg accepts untrusted bytes from any connected peer; this +// target hammers both the length-prefixed path and the raw fallback +// path with arbitrary inputs. +package pex + +import ( + "bytes" + "testing" + + "github.com/nnlgsakib/membuss/internal/wiretest" +) + +const pexMaxFrame = 1 << 20 // mirrors readMsg's internal frame cap + +func FuzzPEXReadMsg(f *testing.F) { + f.Add(append([]byte{0x00, 0x00, 0x00, 0x04}, []byte("abcd")...)) + f.Add([]byte{}) + f.Add([]byte{0x00}) + f.Add(append([]byte{0x7F, 0xFF, 0xFF, 0xFF}, make([]byte, 16)...)) + f.Add([]byte("raw fallback bytes without length prefix")) + + f.Fuzz(func(t *testing.T, data []byte) { + s := wiretest.NewStream(bytes.NewReader(data)) + msg := readMsg(s) + if len(msg) > pexMaxFrame { + t.Fatalf("readMsg returned %d bytes, exceeds cap %d", len(msg), pexMaxFrame) + } + }) +} From 43a304300c08f747d8f585e04b1a9d6f111e3f1f Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:24:41 +0600 Subject: [PATCH 04/16] fix(gateway): reject empty suffix range when size is 0 (GATE-011) parseRange("bytes=-1", size=0) clamped n to 0 and returned (0,0,nil) - a 200 with an empty body instead of a 416. The suffix branch returned before the shared bounds check. Guard added; found by FuzzParseRange (XC-001). Also whitespace/gofmt touch-ups. Co-Authored-By: Claude --- gateway/memgate_v2/memgate.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gateway/memgate_v2/memgate.go b/gateway/memgate_v2/memgate.go index fca8634..57b0ae0 100644 --- a/gateway/memgate_v2/memgate.go +++ b/gateway/memgate_v2/memgate.go @@ -932,7 +932,7 @@ type dagTreeNodeJSON struct { } type dagTreeResponseJSON struct { - Root string `json:"root"` + Root string `json:"root"` Nodes map[string]dagTreeNodeJSON `json:"nodes"` } @@ -1290,6 +1290,9 @@ func parseRange(s string, size int64) (int64, int64, error) { } start = uint64(size) - n end = uint64(size) + if end <= start { + return 0, 0, fmt.Errorf("range out of bounds") + } return int64(start), int64(end), nil } start, err = strconv.ParseUint(startStr, 10, 64) @@ -1925,7 +1928,7 @@ func (m *MemGate) checkBaseRedirect(r *http.Request, root mid.MID, innerPath str if strings.HasSuffix(redirectPath, "index.html") { redirectPath = strings.TrimSuffix(redirectPath, "index.html") } - + // Clean and compare path to see if base path prefix is already at the end cleanedPath := strings.TrimSuffix(redirectPath, "/") if strings.HasSuffix(cleanedPath, "/"+relBase) || cleanedPath == relBase || strings.HasSuffix(cleanedPath, "/"+base) { @@ -2727,4 +2730,3 @@ func (m *MemGate) executeEdgeFunction(w http.ResponseWriter, r *http.Request, co w.WriteHeader(status) _, _ = w.Write([]byte(resp.Body)) } - From d69bd77f7af6e934fe5015ded8d7ba910613a439 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:24:52 +0600 Subject: [PATCH 05/16] bench: baseline benchmark suite (XC-002) Chunker (fixed/Rabin/FastCDC), store Put/Get/batch, erasure encode/decode/verify, gateway Range parsing. Wired into weekly ci-bench workflow with 90-day artifact history. Co-Authored-By: Claude --- core/chunk/chunk_bench_test.go | 56 +++++++++++++++++++++++ core/erasure/erasure_bench_test.go | 61 ++++++++++++++++++++++++++ core/store/store_bench_test.go | 61 ++++++++++++++++++++++++++ gateway/memgate_v2/range_bench_test.go | 25 +++++++++++ 4 files changed, 203 insertions(+) create mode 100644 core/chunk/chunk_bench_test.go create mode 100644 core/erasure/erasure_bench_test.go create mode 100644 core/store/store_bench_test.go create mode 100644 gateway/memgate_v2/range_bench_test.go diff --git a/core/chunk/chunk_bench_test.go b/core/chunk/chunk_bench_test.go new file mode 100644 index 0000000..58a61e4 --- /dev/null +++ b/core/chunk/chunk_bench_test.go @@ -0,0 +1,56 @@ +// Benchmarks for chunker throughput (finding.txt XC-002). +// Run: go test -bench BenchmarkChunker -benchmem ./core/chunk/ +package chunk + +import ( + "bytes" + "io" + "testing" +) + +// benchData returns a deterministic pseudo-random payload so CDC +// boundaries are exercised realistically without rand overhead. +func benchData(n int) []byte { + data := make([]byte, n) + var x uint64 = 0x9e3779b97f4a7c15 + for i := range data { + x ^= x << 13 + x ^= x >> 7 + x ^= x << 17 + data[i] = byte(x) + } + return data +} + +func benchChunker(b *testing.B, f ChunkerFactory) { + const size = 8 << 20 // 8 MiB + data := benchData(size) + b.SetBytes(int64(size)) + b.ResetTimer() + for b.Loop() { + ch, err := f(bytes.NewReader(data)) + if err != nil { + b.Fatal(err) + } + var total int + for { + blk, err := ch.Next() + if err == io.EOF { + break + } + if err != nil { + b.Fatal(err) + } + total += len(blk.Data()) + } + if total != size { + b.Fatalf("chunked %d of %d bytes", total, size) + } + } +} + +func BenchmarkChunkerFixed256K(b *testing.B) { benchChunker(b, NewFixed(DefaultBlockSize)) } + +func BenchmarkChunkerRabin(b *testing.B) { benchChunker(b, NewRabin()) } + +func BenchmarkChunkerFastCDC(b *testing.B) { benchChunker(b, NewFastCDC()) } diff --git a/core/erasure/erasure_bench_test.go b/core/erasure/erasure_bench_test.go new file mode 100644 index 0000000..cccdc02 --- /dev/null +++ b/core/erasure/erasure_bench_test.go @@ -0,0 +1,61 @@ +// Benchmarks for Reed-Solomon encode/decode (finding.txt XC-002). +package erasure + +import ( + "testing" +) + +func benchEncoder(b *testing.B) *Encoder { + enc, err := NewEncoder(DefaultConfig()) + if err != nil { + b.Fatal(err) + } + return enc +} + +func BenchmarkErasureEncode1MiB(b *testing.B) { + enc := benchEncoder(b) + data := make([]byte, 1<<20) + b.SetBytes(1 << 20) + for b.Loop() { + if _, err := enc.Encode(data); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkErasureDecode1MiB(b *testing.B) { + enc := benchEncoder(b) + data := make([]byte, 1<<20) + encd, err := enc.Encode(data) + if err != nil { + b.Fatal(err) + } + shards := make([][]byte, len(encd.Shards)) + for i, sh := range encd.Shards { + shards[i] = sh.Data + } + for b.Loop() { + if _, err := enc.Decode(shards, encd.Manifest); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkErasureVerify1MiB(b *testing.B) { + enc := benchEncoder(b) + data := make([]byte, 1<<20) + encd, err := enc.Encode(data) + if err != nil { + b.Fatal(err) + } + shards := make([][]byte, len(encd.Shards)) + for i, sh := range encd.Shards { + shards[i] = sh.Data + } + for b.Loop() { + if ok, err := enc.Verify(shards); err != nil || !ok { + b.Fatal("verify failed") + } + } +} diff --git a/core/store/store_bench_test.go b/core/store/store_bench_test.go new file mode 100644 index 0000000..bad9e0e --- /dev/null +++ b/core/store/store_bench_test.go @@ -0,0 +1,61 @@ +// Benchmarks for the Pebble-backed MemStore (finding.txt XC-002). +// In-memory Pebble isolates CPU/codec cost from disk variance; +// run against a real path separately when profiling IO. +package store + +import ( + "testing" + + "github.com/nnlgsakib/membuss/core/mid" +) + +func benchStore(b *testing.B) *MemStore { + s, err := NewMemStore(Options{InMemory: true}) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = s.Close() }) + return s +} + +func BenchmarkStorePut256K(b *testing.B) { + s := benchStore(b) + blk := make([]byte, 256<<10) + for b.Loop() { + blk[0] = byte(b.N) + if err := s.Put(mid.FromBytes(blk), blk); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkStoreGet256K(b *testing.B) { + s := benchStore(b) + blk := make([]byte, 256<<10) + blk[0] = 0xAB + m := mid.FromBytes(blk) + if err := s.Put(m, blk); err != nil { + b.Fatal(err) + } + for b.Loop() { + if _, err := s.Get(m); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkStorePutBatch256x64K(b *testing.B) { + s := benchStore(b) + const n = 256 + batch := make([]Block, 0, n) + for i := range n { + data := make([]byte, 64<<10) + data[0] = byte(i) + batch = append(batch, Block{MID: mid.FromBytes(data), Data: data}) + } + for b.Loop() { + if err := s.PutBatch(batch); err != nil { + b.Fatal(err) + } + } +} diff --git a/gateway/memgate_v2/range_bench_test.go b/gateway/memgate_v2/range_bench_test.go new file mode 100644 index 0000000..c6b4baa --- /dev/null +++ b/gateway/memgate_v2/range_bench_test.go @@ -0,0 +1,25 @@ +// Benchmark for Range-header parsing (finding.txt XC-002). +package memgate_v2 + +import "testing" + +func BenchmarkParseRange(b *testing.B) { + specs := []struct { + name string + hdr string + size int64 + }{ + {"suffix", "bytes=-1024", 1 << 20}, + {"open-end", "bytes=4096-", 1 << 20}, + {"closed", "bytes=100-199", 1 << 20}, + } + for _, s := range specs { + b.Run(s.name, func(b *testing.B) { + for b.Loop() { + if _, _, err := parseRange(s.hdr, s.size); err != nil { + b.Fatal(err) + } + } + }) + } +} From 214a191f5aa091178869e851a77675cb9e6552f2 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:25:00 +0600 Subject: [PATCH 06/16] build(make): fuzz targets + portable CGO handling make fuzz / fuzz-all / fuzz-one for the XC-001 targets. Recipes use plain per-target rules and Makefile-level CGO export so they run identically under sh and cmd.exe (PowerShell-spawned make has no sh). RACE=1 flips CGO_ENABLED for -race runs. Co-Authored-By: Claude --- Makefile | 77 +++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 657ee94..dfb0029 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ # (set RACE=1 to enable the race detector where a # C toolchain is available, e.g. `make test RACE=1`) # make lint run golangci-lint (skipped if not installed) +# make fuzz replay fuzz seed corpora + saved counterexamples +# make fuzz-all fuzz every target FUZZTIME each (default 30s) +# make fuzz-one fuzz one target: PKG=./net/pex/ FUZZ=FuzzPEXReadMsg # make run-daemon run the daemon with ./membuss.yaml # make tidy go mod tidy # make clean remove bin/, proto outputs, and frontend build artifacts @@ -30,22 +33,21 @@ else endif GO ?= go +# Exported to child processes; RACE=1 flips it to 1 below (race +# detector needs cgo). export CGO_ENABLED=0 PKG := ./... -# Race detector toggle. The project has no cgo dependency, so it -# builds and tests fine with CGO_ENABLED=0 everywhere. The one thing -# that needs cgo is `go test -race`, which requires a working C -# toolchain. Default is OFF so the suite runs on any machine; opt in -# with `make test RACE=1` where a C compiler is available. +# Race detector toggle. Default OFF so the suite runs on any machine; +# opt in with `make test RACE=1` where a C compiler is available. RACE ?= 0 ifeq ($(RACE),1) TEST_FLAGS := -race -count=1 - TEST_CGO := 1 + CGO_ENABLED := 1 else TEST_FLAGS := -count=1 - TEST_CGO := 0 endif + BUILD_DIR := bin # Single unified binary: membuss is both the node daemon and the # operator CLI (run the node with `membuss daemon start`). @@ -62,7 +64,7 @@ IMAGE ?= ghcr.io/membuss-protocol/membuss:latest CONTAINER ?= membuss COMPOSE ?= docker compose -.PHONY: build frontend frontend-dev proto test lint run-daemon tidy clean \ +.PHONY: build frontend frontend-dev proto test lint fuzz fuzz-all fuzz-one run-daemon tidy clean \ docker-build docker-run docker-stop docker-logs docker-push \ docker-compose-up docker-compose-down docker-compose-logs @@ -93,7 +95,7 @@ else endif test: - CGO_ENABLED=$(TEST_CGO) $(GO) test $(PKG) $(TEST_FLAGS) + $(GO) test $(PKG) $(TEST_FLAGS) lint: @if command -v golangci-lint >/dev/null 2>&1; then \ @@ -156,3 +158,60 @@ docker-compose-down: docker-compose-logs: $(COMPOSE) logs -f + +# --------------------------------------------------------------------------- +# Fuzzing (finding.txt XC-001) +# --------------------------------------------------------------------------- +# +# Go runs ONE fuzz target per command, so fuzz-all chains one rule +# per target. Targets: +# +# make fuzz replay seed corpus + saved +# counterexamples (fast, no mutation) +# make fuzz-all actually fuzz every target for +# FUZZTIME each (default 30s) +# make fuzz-one PKG=... FUZZ=... fuzz a single target +# +# A crash writes a counterexample to /testdata/fuzz//; +# `make fuzz` then replays it forever after as a regression test. + +FUZZTIME ?= 30s + +.PHONY: fuzz fuzz-all fuzz-one \ + fuzz-parse-range fuzz-pex fuzz-memex-frame fuzz-descriptor fuzz-dht-ns fuzz-keyring + +fuzz: + $(GO) test -run 'Fuzz' ./gateway/memgate_v2/ ./net/pex/ ./net/memex_v2/ ./core/descriptor/ ./net/dht/ ./core/keyring/ + +# One rule per target (Go fuzzes one function per process). Plain +# single-line recipes so they run identically under sh and cmd.exe +# (PowerShell-spawned make has no sh). +fuzz-all: fuzz-parse-range fuzz-pex fuzz-memex-frame fuzz-descriptor fuzz-dht-ns fuzz-keyring + +fuzz-parse-range: + @echo == FuzzParseRange gateway/memgate_v2 $(FUZZTIME) == + $(GO) test -run '^FuzzParseRange$$' -fuzz '^FuzzParseRange$$' -fuzztime $(FUZZTIME) ./gateway/memgate_v2/ + +fuzz-pex: + @echo == FuzzPEXReadMsg net/pex $(FUZZTIME) == + $(GO) test -run '^FuzzPEXReadMsg$$' -fuzz '^FuzzPEXReadMsg$$' -fuzztime $(FUZZTIME) ./net/pex/ + +fuzz-memex-frame: + @echo == FuzzMemexReadFrame net/memex_v2 $(FUZZTIME) == + $(GO) test -run '^FuzzMemexReadFrame$$' -fuzz '^FuzzMemexReadFrame$$' -fuzztime $(FUZZTIME) ./net/memex_v2/ + +fuzz-descriptor: + @echo == FuzzDescriptorParse core/descriptor $(FUZZTIME) == + $(GO) test -run '^FuzzDescriptorParse$$' -fuzz '^FuzzDescriptorParse$$' -fuzztime $(FUZZTIME) ./core/descriptor/ + +fuzz-dht-ns: + @echo == FuzzMemNSValidate net/dht $(FUZZTIME) == + $(GO) test -run '^FuzzMemNSValidate$$' -fuzz '^FuzzMemNSValidate$$' -fuzztime $(FUZZTIME) ./net/dht/ + +fuzz-keyring: + @echo == FuzzKeyImportParse core/keyring $(FUZZTIME) == + $(GO) test -run '^FuzzKeyImportParse$$' -fuzz '^FuzzKeyImportParse$$' -fuzztime $(FUZZTIME) ./core/keyring/ + +# Single target, e.g.: make fuzz-one PKG=./net/pex/ FUZZ=FuzzPEXReadMsg +fuzz-one: + $(GO) test -run '^$(FUZZ)$$' -fuzz '^$(FUZZ)$$' -fuzztime $(FUZZTIME) $(PKG) From 3f56bae10d9c2fb0e7741bb1f2bd76b63d49595e Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:25:16 +0600 Subject: [PATCH 07/16] ci: GitHub Actions pipeline (XC-003) lint (reporting-only), govulncheck (reporting-only, see XC-012), test matrix with -race on unix, coverage report + artifact, 6-target cross-build matrix, desktop submodule contract build. Shared build-frontend composite action feeds the go:embed'd explorer dist on every Go job. Weekly benchmark workflow included. Verified locally with act: modules job green; lint and govulncheck functional end-to-end. Co-Authored-By: Claude --- .github/actions/build-frontend/action.yml | 25 +++ .github/workflows/ci-bench.yml | 37 +++++ .github/workflows/ci.yml | 184 ++++++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 .github/actions/build-frontend/action.yml create mode 100644 .github/workflows/ci-bench.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/actions/build-frontend/action.yml b/.github/actions/build-frontend/action.yml new file mode 100644 index 0000000..b5e5777 --- /dev/null +++ b/.github/actions/build-frontend/action.yml @@ -0,0 +1,25 @@ +name: Build explorer frontend +description: >- + Builds the explorer-web SvelteKit app into gateway/explorer/web/dist, + which gateway/explorer embeds via //go:embed all:web/dist. + Required before any `go build`/`go test` of the root module in CI, + because dist/ is gitignored. +runs: + using: composite + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: explorer-web/package-lock.json + + - name: Install dependencies + shell: bash + working-directory: explorer-web + run: npm ci + + - name: Build frontend + shell: bash + working-directory: explorer-web + run: npm run build diff --git a/.github/workflows/ci-bench.yml b/.github/workflows/ci-bench.yml new file mode 100644 index 0000000..101cbb6 --- /dev/null +++ b/.github/workflows/ci-bench.yml @@ -0,0 +1,37 @@ +# XC-002: benchmark suite CI wiring. Manual or weekly run; +# results land as artifacts for benchstat comparison across runs. +name: Benchmarks + +on: + workflow_dispatch: + schedule: + - cron: '23 4 * * 1' # weekly Monday + +permissions: + contents: read + +jobs: + bench: + name: go test -bench + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build explorer frontend + uses: ./.github/actions/build-frontend + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run benchmarks + run: | + go test -run '^$' -bench . -benchmem -benchtime=2s ./core/chunk/ ./core/store/ ./core/erasure/ ./gateway/memgate_v2/ | tee bench.txt + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: bench-${{ github.run_number }} + path: bench.txt + retention-days: 90 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3a1a634 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,184 @@ +name: CI + +on: + push: + branches: [master, features] + pull_request: + branches: [master] + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + lint: + name: Lint (golangci-lint) + # Reporting-only until the pre-existing staticcheck/unused debt + # is burned down (finding.txt XC-004 burn-down list). + continue-on-error: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build explorer frontend + uses: ./.github/actions/build-frontend + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - uses: golangci/golangci-lint-action@v8 + with: + version: latest + args: --timeout=10m + + govulncheck: + name: Vulnerabilities (govulncheck) + # Reporting-only until the 14 called vulnerabilities (5 modules + + # stdlib, found 2026-08-22) are triaged: Go toolchain bump to + # 1.25.13, grpc v1.82.1, x/text v0.39.0, pion/dtls v3.1.4, + # webtransport-go v0.11.1. One finding has no fix available yet. + continue-on-error: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run govulncheck + # @latest: outside-module invocation; plain `go run pkg` would + # error "no required module provides package". + run: go run golang.org/x/vuln/cmd/govulncheck@latest ./... + + test: + name: Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - name: Build explorer frontend + uses: ./.github/actions/build-frontend + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Test (race on unix, plain on windows) + shell: bash + run: | + if [ "${{ matrix.os }}" = "windows-latest" ]; then + go test ./... + else + CGO_ENABLED=1 go test -race -timeout 20m ./... + fi + + coverage: + name: Coverage report + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build explorer frontend + uses: ./.github/actions/build-frontend + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Test with coverage + run: go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out ./... + + - name: Coverage summary + if: always() + run: go run golang.org/x/tools/cmd/cover@latest -html=coverage.out -o coverage.html; go tool cover -func=coverage.out | tail -1 | tee -a "$GITHUB_STEP_SUMMARY" || true + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-report + path: | + coverage.out + coverage.html + retention-days: 7 + + build: + name: Build ${{ matrix.goos }}-${{ matrix.goarch }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { goos: linux, goarch: amd64 } + - { goos: linux, goarch: arm64 } + - { goos: darwin, goarch: amd64 } + - { goos: darwin, goarch: arm64 } + - { goos: windows, goarch: amd64 } + - { goos: windows, goarch: arm64 } + steps: + - uses: actions/checkout@v4 + + - name: Build explorer frontend + uses: ./.github/actions/build-frontend + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build daemon binary + shell: bash + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: 0 + run: | + go build -trimpath -ldflags "-s -w" -o "bin/membuss$( [ "${{ matrix.goos }}" = "windows" ] && echo '.exe' )" ./cmd/membuss + + - name: Smoke-test cross-compiled binary (native only) + if: matrix.goos == 'linux' && matrix.goarch == 'amd64' + run: ./bin/membuss --help > /dev/null && echo "binary runs" + + modules: + name: Build submodule ${{ matrix.module }} + # XC-005: root API changes must not silently break the desktop + # module (replace => ../). memgit/memploy are separate projects, + # NOT part of this repo — never add them here. + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: [desktop] + defaults: + run: + working-directory: ${{ matrix.module }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: ${{ matrix.module }}/go.mod + cache: true + cache-dependency-path: ${{ matrix.module }}/go.sum + + - name: Build desktop frontend (wails embed target) + working-directory: ${{ matrix.module }}/frontend + run: | + npm ci + npm run build + + - run: go build ./... + - run: go vet ./... From b2b98d71a0d438c1a752c8d4718c8978b13bb5f1 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:25:28 +0600 Subject: [PATCH 08/16] ci: nightly 3-node chaos workflow (XC-007) compose swarm, seed/verify round-trip, kill the seed node and require refetch via remaining peer. Toxiproxy packet-loss profiles still open (see finding.txt XC-007 note). Co-Authored-By: Claude --- .github/workflows/ci-chaos.yml | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/ci-chaos.yml diff --git a/.github/workflows/ci-chaos.yml b/.github/workflows/ci-chaos.yml new file mode 100644 index 0000000..653d06f --- /dev/null +++ b/.github/workflows/ci-chaos.yml @@ -0,0 +1,66 @@ +# XC-007: multi-node chaos/e2e matrix (nightly). +# v1 chaos = node-kill mid-transfer on a 3-node swarm. +# Network-level packet loss/latency (toxiproxy) tracked in finding.txt. +name: Chaos E2E + +on: + workflow_dispatch: + schedule: + - cron: '41 3 * * *' # nightly + +permissions: + contents: read + +jobs: + chaos: + name: 3-node swarm survives node kill + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build daemon binary (linux/amd64) + run: go build -trimpath -o membuss ./cmd/membuss + + - name: Build container image + run: docker build -t membuss:local . + + - name: Start 3-node swarm + run: docker compose -f docker-compose.multi.yml up -d --wait || docker compose -f docker-compose.multi.yml ps + + - name: Wait for nodes to settle + run: sleep 20 + + - name: Seed content on node 1, fetch from node 2 + id: happy + run: | + set -e + head -c 1048576 /dev/urandom > /tmp/chaos.bin + docker cp /tmp/chaos.bin membuss-1:/tmp/chaos.bin + OUT=$(docker exec membuss-1 /usr/local/bin/membuss add /tmp/chaos.bin) + MID=$(echo "$OUT" | grep -oE '[a-z0-9]{59,}' | head -1) + echo "mid=$MID" >> "$GITHUB_OUTPUT" + sleep 10 # announce + propagation + docker exec membuss-2 /usr/local/bin/membuss cat "$MID" > /tmp/out.bin + cmp /tmp/out.bin /tmp/chaos.bin + + + - name: Chaos — kill node 1 (seed node) + if: steps.happy.outcome == 'success' + run: docker stop membuss-1 + + - name: Refetch from node 2 after seed death + if: steps.happy.outcome == 'success' + run: | + set -e + MID="${{ steps.happy.outputs.mid }}" + docker exec membuss-2 /usr/local/bin/membuss cat "$MID" > /tmp/out2.bin + cmp /tmp/out2.bin /tmp/chaos.bin + + - name: Teardown (always) + if: always() + run: docker compose -f docker-compose.multi.yml down -v || true From 30527aa5f11ad52cef0d41571e9528846feda0e8 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:25:42 +0600 Subject: [PATCH 09/16] test(config): desktop submodule contract tests (XC-005) Pins the two touchpoints desktop/ has with this module: Default()->yaml->Load round-trip and core/version.Version. Breaks in the root module CI instead of silently breaking the desktop release build. Co-Authored-By: Claude --- config/contract_desktop_test.go | 71 +++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 config/contract_desktop_test.go diff --git a/config/contract_desktop_test.go b/config/contract_desktop_test.go new file mode 100644 index 0000000..d312e4e --- /dev/null +++ b/config/contract_desktop_test.go @@ -0,0 +1,71 @@ +// Desktop submodule contract (finding.txt XC-005). +// +// desktop/ imports this package through `replace => ../` and does +// exactly two things with it: +// 1. WriteDefaultConfig: config.Default() -> yaml.Marshal -> write config.yaml +// 2. links core/version.Version +// +// This test pins that surface. If a change here breaks the desktop +// build or the config files it writes, this fails first — in the +// root module, where CI runs it — instead of silently breaking the +// desktop release build. +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/nnlgsakib/membuss/core/version" + "gopkg.in/yaml.v3" +) + +// TestDesktopContractDefaultRoundTrip mirrors desktop WriteDefaultConfig: +// Default() -> yaml.Marshal -> file -> Load must preserve the fields the +// daemon needs to boot (addresses, data dir). +func TestDesktopContractDefaultRoundTrip(t *testing.T) { + cfg := Default() + if cfg == nil { + t.Fatal("config.Default() returned nil") + } + if cfg.APIAddr == "" || cfg.GatewayAddr == "" || cfg.GRPCAddr == "" { + t.Fatalf("Default() missing listen addrs: api=%q gw=%q grpc=%q", + cfg.APIAddr, cfg.GatewayAddr, cfg.GRPCAddr) + } + cfg.DataDir = filepath.ToSlash(filepath.Join(t.TempDir(), "node")) + + data, err := yaml.Marshal(cfg) + if err != nil { + t.Fatalf("yaml.Marshal(Default()): %v", err) + } + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + back, err := Load(path) + if err != nil { + t.Fatalf("Load(marshaled default): %v", err) + } + checks := []struct { + name string + got, want string + }{ + {"DataDir", back.DataDir, cfg.DataDir}, + {"APIAddr", back.APIAddr, "127.0.0.1:5001"}, + {"GatewayAddr", back.GatewayAddr, "127.0.0.1:8080"}, + {"GRPCAddr", back.GRPCAddr, "127.0.0.1:50051"}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("%s = %q, want %q (round-trip drift)", c.name, c.got, c.want) + } + } +} + +// TestDesktopContractVersionSymbol pins the version variable the desktop +// UI displays and update-checks against. +func TestDesktopContractVersionSymbol(t *testing.T) { + if version.Version == "" { + t.Fatal("core/version.Version is empty") + } +} From 58317e6e76cc07cff2b313e3f77f679f3a679b44 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:25:43 +0600 Subject: [PATCH 10/16] docs: wire protocol specs (XC-006) memex v2 framing, PEX, descriptor layout (sha256 trailer MUST verify), DHT namespaces + validators, edge RPC surface. Index with versioning plan (new protocol IDs, deprecation window, Hello-frame negotiation). Co-Authored-By: Claude --- docs/spec/README.md | 25 +++++++++++++++++++++++ docs/spec/descriptor.md | 20 +++++++++++++++++++ docs/spec/dht-namespaces.md | 21 +++++++++++++++++++ docs/spec/edge-rpc.md | 32 +++++++++++++++++++++++++++++ docs/spec/memex-v2.md | 40 +++++++++++++++++++++++++++++++++++++ docs/spec/pex.md | 35 ++++++++++++++++++++++++++++++++ 6 files changed, 173 insertions(+) create mode 100644 docs/spec/README.md create mode 100644 docs/spec/descriptor.md create mode 100644 docs/spec/dht-namespaces.md create mode 100644 docs/spec/edge-rpc.md create mode 100644 docs/spec/memex-v2.md create mode 100644 docs/spec/pex.md diff --git a/docs/spec/README.md b/docs/spec/README.md new file mode 100644 index 0000000..ff6786a --- /dev/null +++ b/docs/spec/README.md @@ -0,0 +1,25 @@ +# Membuss Wire Protocol Specifications + +Normative reference for every peer-to-peer protocol membuss nodes speak. +Each doc states: protocol ID, framing, message schema, limits, and the +versioning/negotiation plan for a future v3. + +| Doc | Protocol ID | +|-----|-------------| +| [memex-v2.md](memex-v2.md) | `/membuss/memex/2.0.0` (+ `/membuss/memex-bloom/2.0.0`) | +| [pex.md](pex.md) | `/membuss/pex/1.0.0` | +| [dht-namespaces.md](dht-namespaces.md) | `/membuss/dht/1.0.0` (kad record validator namespaces) | +| [edge-rpc.md](edge-rpc.md) | `/membuss/edge/exec/v1` | +| [descriptor.md](descriptor.md) | n/a — `.mbuss` container format | + +## Versioning plan + +Current protocols carry their version inside the protocol ID string +(`/2.0.0`, `/1.0.0`). A breaking v3 must: + +1. Register a new protocol ID (`/membuss/memex/3.0.0`); never mutate an + existing one. +2. Keep the old handler mounted during the deprecation window. +3. Prefer capability negotiation via a `Hello` frame once any protocol + gains per-connection options; until then the protocol-ID match IS the + contract. diff --git a/docs/spec/descriptor.md b/docs/spec/descriptor.md new file mode 100644 index 0000000..376be45 --- /dev/null +++ b/docs/spec/descriptor.md @@ -0,0 +1,20 @@ +# `.mbuss` descriptor container + +Defined in `core/descriptor/descriptor.go` (`Serialize`/`Parse`). + +``` ++--------+---------+-------------------+---------------+ +| "MEMB" | version | protobuf payload | sha256 | +| 4 B | 1 B | (variable) | trailer 32 B | ++--------+---------+-------------------+---------------+ +``` + +- Magic: ASCII `MEMB`. +- Version byte: only the current value is accepted (`errBadVersion`); + unknown versions must be rejected, never skipped. +- Payload: `membuss.v1.DescriptorPayload` — root MID, total size, block + count, name, mime type, created-at, chunker id/size, bootstrap peers, + MemNS name, signature, block list (MID+size+index each), optional + erasure info (data/parity shard counts, shard MIDs). +- Trailer: SHA-256 over exactly the payload bytes. Parsers MUST verify + it before trusting anything else (finding.txt CORE-I1 hardening list). diff --git a/docs/spec/dht-namespaces.md b/docs/spec/dht-namespaces.md new file mode 100644 index 0000000..ab1ee77 --- /dev/null +++ b/docs/spec/dht-namespaces.md @@ -0,0 +1,21 @@ +# DHT namespaces and record validators + +Routing: `/membuss/dht/1.0.0` (`net/dht/dht.go:ProtocolPrefix`). + +## Validator table + +| Namespace | Validator | Notes | +|-----------|-----------|-------| +| `/memns/` | `validateMemNS` (validator.go) | signed seq records; name = `k` + base36(sha256(owner pubkey)) | +| `/membuss/` | `validateMembuss` | permissive; dev/testing | +| `/membuss/anchors/v1` | registry validator | UNSIGNED today — poisoning risk, finding.txt NETPKG-040 | +| `/membuss/relays/v1` | registry validator | UNSIGNED today — finding.txt NETPKG-040 | + +## Record rules (MemNS) + +- `sequence > 0`, monotonic per name. +- `validity` = unix-nano expiry; expired records rejected. +- Ed25519 signature over `value || seq(u64 BE) || validity(u64 BE)`. +- Signer must be owner (pubkey match) or listed delegate. +- Name binding: sha256(owner pubkey) rendered base36 with `k` prefix must + equal the key after the `/memns/` prefix. diff --git a/docs/spec/edge-rpc.md b/docs/spec/edge-rpc.md new file mode 100644 index 0000000..cd24e1d --- /dev/null +++ b/docs/spec/edge-rpc.md @@ -0,0 +1,32 @@ +# edge RPC — remote function execution + +Protocol ID: `/membuss/edge/exec/v1` (`net/edge_rpc/protocol.go:ProtocolID`). + +## Framing + +JSON messages over libp2p msgio length-delimited streams. This violates +the protobuf-everywhere convention and is tracked for migration +(finding.txt NETPKG-030). + +## Messages + +`RPCRequest`: `mid`, `path`, `code` (WASM/JS bytes), `runtime` +(go/wasm or js), `req` (`memedge.Request`), `limits`. + +`RPCResponse`: `response` (`memedge.Response`), `peer_id`, `tier`, +`error`. + +## Limits and security posture (current) + +- Token bucket: 20 req/s per peer. +- Hardcoded 10s exec timeout (finding.txt NETPKG-031). +- **Unauthenticated execution** — any connected peer may run code on a + node. Default-off allowlist + signed authorization is the planned fix + (finding.txt NETPKG-030). Do not expose nodes running this protocol + to untrusted swarms until that lands. + +## Tiering + +Requests fall through publisher tiers: gateway (`TierPublisher`) then +peer swarm (`TierPeer`); status >= 500 should retry next tier +(finding.txt NETPKG-031 tracks the current gap). diff --git a/docs/spec/memex-v2.md b/docs/spec/memex-v2.md new file mode 100644 index 0000000..c983bee --- /dev/null +++ b/docs/spec/memex-v2.md @@ -0,0 +1,40 @@ +# memex v2 — block exchange protocol + +Protocol ID: `/membuss/memex/2.0.0` (`net/memex_v2/memex.go:ProtocolID`). +Sidecar: `/membuss/memex-bloom/2.0.0` (`bloom.go:BloomProtocolID`). + +## Framing + +Every frame on a memex stream: + +``` ++--------------+----------------------+ +| len: 4 bytes | protobuf payload | +| big-endian | (len bytes) | ++--------------+----------------------+ +``` + +- Length prefix: `uint32` big-endian (`readFrame`, memex.go). +- `len == 0` or `len > 16 MiB` (`maxFrameSize = 16 << 20`) → frame invalid, + sender must reset the stream. Receiver returns `nil` and tears down. + +## Messages + +All payloads are `membuss.v1.MemexMessage` (`proto/membuss.pb.go`): + +| Field | Type | Meaning | +|-------|------|---------| +| `wants` | `repeated WantEntry` | blocks the sender requests | +| `blocks` | `repeated Block` | payload blocks answering wants | +| `object_infos` | `map` | metadata sidecar keyed by MID | +| `sequence_number` | `uint64` | session ordering, monotonic per peer | + +`WantEntry`: `mid`, `priority` (int32; currently unscheduled — see +finding.txt NETPKG-022), `send_dont_have`, `want_type`. + +## Limits + +- Frame cap: 16 MiB. +- Sessions idle out after 60s without activity. +- AIMD congestion window: init 8, cap 128, halve on write error + (see finding.txt NETPKG-003/014 for known gaps). diff --git a/docs/spec/pex.md b/docs/spec/pex.md new file mode 100644 index 0000000..9e474c4 --- /dev/null +++ b/docs/spec/pex.md @@ -0,0 +1,35 @@ +# PEX — peer exchange protocol + +Protocol ID: `/membuss/pex/1.0.0` (`net/pex/pex.go:ProtocolID`). + +## Framing + +``` ++--------------+----------------------+ +| len: 4 bytes | protobuf payload | +| big-endian | (len bytes) | ++--------------+----------------------+ +``` + +- Cap: 1 MiB (`readMsg`, pex.go). +- **Legacy fallback**: if the first read yields fewer than 4 bytes the + receiver abandons framing and buffers until stream EOF or 1 MiB. + Known-broken on short reads (finding.txt NETPKG-011); treat as + deprecated and do not rely on it for new clients. + +## Messages + +Payload is `membuss.v1.PeerInfo`, optionally wrapped in a signed +gossip record: + +| Field | Meaning | +|-------|---------| +| `peer_id` | libp2p peer ID string | +| `addrs` / `relay_addrs` | direct + relay multiaddrs (Phase 12) | +| `last_seen` | unix seconds | +| `reachability` | enum, sender's observed NAT posture | +| `last_dial_success` | bool | +| `signature` / `pub_key` / `seq` | Ed25519-signed record; monotonic `seq` gates replays (Phase 20) | + +Anti-entropy: peers exchange signed records both directions per gossip +round; `(seq, lastSeen)` ordering decides winners. From 6c0dffa32021e66aa417340ef9945a445268372d Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:25:57 +0600 Subject: [PATCH 11/16] feat(audit): append-only audit trail for destructive admin ops (XC-008) JSONL at /logs/audit.jsonl with 10 MiB rotation. Explorer delete/flash/keyring-rm handlers record actor (client IP), action, target. Node API exposes GET /api/v1/audit (last 100 entries, API-key gated). Daemon opens one logger and injects it into both servers. Co-Authored-By: Claude --- api/api.go | 23 ++++++ cmd/membuss/daemon/main.go | 33 ++++++--- core/audit/audit.go | 137 +++++++++++++++++++++++++++++++++++ core/audit/audit_test.go | 42 +++++++++++ gateway/explorer/audit.go | 23 ++++++ gateway/explorer/explorer.go | 72 +++++++++--------- 6 files changed, 287 insertions(+), 43 deletions(-) create mode 100644 core/audit/audit.go create mode 100644 core/audit/audit_test.go create mode 100644 gateway/explorer/audit.go diff --git a/api/api.go b/api/api.go index 7e5c344..0f56adb 100644 --- a/api/api.go +++ b/api/api.go @@ -29,6 +29,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/nnlgsakib/membuss/core/audit" "github.com/nnlgsakib/membuss/core/keyring" "github.com/nnlgsakib/membuss/core/memedge" "github.com/nnlgsakib/membuss/core/memns" @@ -204,6 +205,9 @@ type Config struct { APIKey string // Metrics, if non-nil, is exposed at GET /metrics. Metrics *metrics.Metrics + // Audit, if non-nil, serves GET /audit (last 100 entries of the + // destructive-operation trail). + Audit *audit.Logger // Phase 18: MemNS and KeyRing fields KeyRing *keyring.KeyRing @@ -299,6 +303,7 @@ func (a *NodeAPI) buildRouter() chi.Router { r.Get("/peers", a.handlePeers) r.Get("/node/info", a.handleNodeInfo) r.Post("/gc", a.handleGC) + r.Get("/audit", a.handleAudit) r.Delete("/delete/{mid}", a.handleDelete) r.Get("/healthz", a.handleHealthz) @@ -714,6 +719,24 @@ func (a *NodeAPI) handleGC(w http.ResponseWriter, r *http.Request) { }) } +// handleAudit serves the last 100 entries of the destructive-operation +// audit trail (XC-008). +func (a *NodeAPI) handleAudit(w http.ResponseWriter, r *http.Request) { + if a.cfg.Audit == nil { + fail(w, http.StatusNotImplemented, fmt.Errorf("audit log not configured")) + return + } + entries, err := a.cfg.Audit.Tail(100) + if err != nil { + fail(w, http.StatusInternalServerError, err) + return + } + if entries == nil { + entries = []audit.Entry{} + } + ok(w, entries) +} + func (a *NodeAPI) handleDelete(w http.ResponseWriter, r *http.Request) { midStr := chi.URLParam(r, "mid") if midStr == "" { diff --git a/cmd/membuss/daemon/main.go b/cmd/membuss/daemon/main.go index 66d66a1..03f5caa 100644 --- a/cmd/membuss/daemon/main.go +++ b/cmd/membuss/daemon/main.go @@ -49,6 +49,7 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" + "github.com/nnlgsakib/membuss/core/audit" "github.com/nnlgsakib/membuss/core/db" "github.com/nnlgsakib/membuss/core/ipc" @@ -58,12 +59,12 @@ import ( "github.com/nnlgsakib/membuss/core/memedge" "github.com/nnlgsakib/membuss/core/memlink" "github.com/nnlgsakib/membuss/core/memns" - "github.com/nnlgsakib/membuss/net/edge_rpc" "github.com/nnlgsakib/membuss/core/mid" "github.com/nnlgsakib/membuss/core/shard" "github.com/nnlgsakib/membuss/core/store" "github.com/nnlgsakib/membuss/core/version" "github.com/nnlgsakib/membuss/net/dht" + "github.com/nnlgsakib/membuss/net/edge_rpc" "github.com/nnlgsakib/membuss/net/herald" "github.com/nnlgsakib/membuss/net/host" memex "github.com/nnlgsakib/membuss/net/memex_v2" @@ -163,6 +164,16 @@ func Run(args []string) error { mtrx = metrics.New() } + // XC-008: append-only audit trail for destructive admin + // operations (delete / flash / keyring rm), served via the + // API /audit endpoint. + aud, err := audit.Open(cfg.DataDir) + if err != nil { + logger.Error("audit", "err", err.Error()) + os.Exit(1) + } + defer aud.Close() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -263,9 +274,9 @@ func Run(args []string) error { } hostCfg := host.Config{ - ListenAddrs: cfg.ListenAddrs, - AnnounceAddrs: cfg.AnnounceAddrs, - DataDir: cfg.DataDir, + ListenAddrs: cfg.ListenAddrs, + AnnounceAddrs: cfg.AnnounceAddrs, + DataDir: cfg.DataDir, UserAgent: func() string { ua := "membuss/v" + version.Version commit := version.GitCommit @@ -693,7 +704,7 @@ func Run(args []string) error { var gateSrv *httpServer if cfg.Servers.Gateway.Enabled { - srv, err := startGateway(cfg.GatewayAddr, newMemgateAdapter(backend), newExplorerAdapter(backend, cfg.AnchorMode, kr, memnsRes), cfg.GatewayRateLimitPerMin, cfg.GatewayTLS, memnsRes, cfg.DataDir, cfg.LogLevel, tunMgr, gateHTTPReg, cfg.MetricsToken, edgeEngine, edgeSvc) + srv, err := startGateway(cfg.GatewayAddr, newMemgateAdapter(backend), newExplorerAdapter(backend, cfg.AnchorMode, kr, memnsRes), cfg.GatewayRateLimitPerMin, cfg.GatewayTLS, memnsRes, cfg.DataDir, cfg.LogLevel, aud, tunMgr, gateHTTPReg, cfg.MetricsToken, edgeEngine, edgeSvc) if err != nil { logger.Error("gateway", "err", err.Error()) os.Exit(1) @@ -708,7 +719,7 @@ func Run(args []string) error { // 10) Node API: local control plane over HTTP/JSON. var apiSrv *httpServer if cfg.Servers.NodeAPI.Enabled { - srv, err := startNodeAPI(cfg.APIAddr, newAPIAdapter(backend), mtrx, cfg.APIKey, cfg.APITLS, kr, memnsRes, cfg.DataDir, cfg.LogLevel, nodeHTTPReg, edgeEngine, edgeSvc, httpIPCLis) + srv, err := startNodeAPI(cfg.APIAddr, newAPIAdapter(backend), mtrx, cfg.APIKey, cfg.APITLS, kr, memnsRes, cfg.DataDir, cfg.LogLevel, aud, nodeHTTPReg, edgeEngine, edgeSvc, httpIPCLis) if err != nil { logger.Error("api", "err", err.Error()) os.Exit(1) @@ -1094,11 +1105,11 @@ func (s *serverGRPC) Stop() { s.gsrv.Stop() } // rateLimitPerMin is the per-IP request budget enforced on // every public request. tls enables HTTPS when its // CertFile/KeyFile are set. -func startGateway(addr string, b memgate.Backend, exp *explorerAdapter, rateLimitPerMin int, tlsCfg config.TLSConfig, memnsRes *memns.Resolver, dataDir string, logLevel string, tunMgr *tunnel.Manager, pluginReg *plugin.MapHTTPRegistry, metricsToken string, edgeEngine memedge.Engine, edgeSvc *edge_rpc.Service) (*httpServer, error) { +func startGateway(addr string, b memgate.Backend, exp *explorerAdapter, rateLimitPerMin int, tlsCfg config.TLSConfig, memnsRes *memns.Resolver, dataDir string, logLevel string, aud *audit.Logger, tunMgr *tunnel.Manager, pluginReg *plugin.MapHTTPRegistry, metricsToken string, edgeEngine memedge.Engine, edgeSvc *edge_rpc.Service) (*httpServer, error) { mg, err := memgate.New(memgate.Config{ Backend: b, MaxCacheBytes: 64 << 20, // 64 MiB LRU - ExplorerHandler: buildExplorer(exp, tunMgr, edgeEngine, edgeSvc), + ExplorerHandler: buildExplorer(exp, aud, tunMgr, edgeEngine, edgeSvc), RateLimitPerMin: rateLimitPerMin, MemNSResolver: memnsRes, LogLevel: logLevel, @@ -1130,12 +1141,13 @@ func startGateway(addr string, b memgate.Backend, exp *explorerAdapter, rateLimi // startNodeAPI brings up the local Node control API. mtrx // exposes Prometheus at /metrics; apiKey enables X-Membuss-Key // auth on every /api/v1 endpoint; tls enables HTTPS. -func startNodeAPI(addr string, b api.Backend, mtrx *metrics.Metrics, apiKey string, tlsCfg config.TLSConfig, keyring *keyring.KeyRing, memnsRes *memns.Resolver, dataDir string, logLevel string, pluginReg *plugin.MapHTTPRegistry, edgeEngine memedge.Engine, edgeSvc *edge_rpc.Service, extraLis ...net.Listener) (*httpServer, error) { +func startNodeAPI(addr string, b api.Backend, mtrx *metrics.Metrics, apiKey string, tlsCfg config.TLSConfig, keyring *keyring.KeyRing, memnsRes *memns.Resolver, dataDir string, logLevel string, aud *audit.Logger, pluginReg *plugin.MapHTTPRegistry, edgeEngine memedge.Engine, edgeSvc *edge_rpc.Service, extraLis ...net.Listener) (*httpServer, error) { nodeAPI, err := api.New(api.Config{ Backend: b, MaxUploadBytes: 1 << 30, // 1 GiB APIKey: apiKey, Metrics: mtrx, + Audit: aud, KeyRing: keyring, MemNSResolver: memnsRes, LogLevel: logLevel, @@ -1278,12 +1290,13 @@ func (h *httpServer) Addr() string { // buildExplorer constructs the explorer http.Handler. // It returns nil when exp is nil so the gateway can be // constructed without an explorer for tests. -func buildExplorer(exp *explorerAdapter, tunMgr *tunnel.Manager, edgeEngine memedge.Engine, edgeSvc *edge_rpc.Service) http.Handler { +func buildExplorer(exp *explorerAdapter, aud *audit.Logger, tunMgr *tunnel.Manager, edgeEngine memedge.Engine, edgeSvc *edge_rpc.Service) http.Handler { if exp == nil { return nil } h, err := explorerPkg.New(explorerPkg.Config{ Backend: exp, + Audit: aud, TunnelManager: tunMgr, EdgeEngine: edgeEngine, EdgeService: edgeSvc, diff --git a/core/audit/audit.go b/core/audit/audit.go new file mode 100644 index 0000000..0e97dd3 --- /dev/null +++ b/core/audit/audit.go @@ -0,0 +1,137 @@ +// Package audit provides an append-only JSONL audit trail for +// destructive administrative operations (finding.txt XC-008): +// content deletes, node flash (DropAll), keyring removals. +// +// The log lives at /logs/audit.jsonl. Every entry records +// who (client IP / peer), when, what action, and the target. +package audit + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + "time" +) + +// Entry is one audited event. +type Entry struct { + Time time.Time `json:"time"` + Actor string `json:"actor"` // client IP or peer ID + Action string `json:"action"` // e.g. "delete", "drop_all" + Target string `json:"target,omitempty"` + Detail map[string]string `json:"detail,omitempty"` +} + +// Logger appends entries to the audit JSONL file. +// A nil *Logger is valid and discards everything, so call sites +// never need nil checks. +type Logger struct { + mu sync.Mutex + path string + f *os.File +} + +const maxLogBytes = 10 << 20 // rotate past 10 MiB + +// Open creates (or appends to) the audit log under dataDir/logs/. +func Open(dataDir string) (*Logger, error) { + if filepath.Clean(dataDir) == "" { + return nil, errors.New("audit: empty datadir") + } + dir := filepath.Join(dataDir, "logs") + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + path := filepath.Join(dir, "audit.jsonl") + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + if err != nil { + return nil, err + } + return &Logger{path: path, f: f}, nil +} + +// Log appends one entry. Best-effort by design: audit failures must +// not break the destructive operation itself, but the error is +// returned so callers can log it. +func (l *Logger) Log(actor, action, target string, detail map[string]string) error { + if l == nil { + return nil + } + e := Entry{Time: time.Now().UTC(), Actor: actor, Action: action, Target: target, Detail: detail} + b, err := json.Marshal(e) + if err != nil { + return err + } + b = append(b, '\n') + + l.mu.Lock() + defer l.mu.Unlock() + if l.f == nil { + return errors.New("audit: closed") + } + if st, serr := l.f.Stat(); serr == nil && st.Size()+int64(len(b)) > maxLogBytes { + l.rotateLocked() + } + if _, err := l.f.Write(b); err != nil { + return err + } + return nil +} + +// rotateLocked renames the current log aside and reopens a fresh one. +// The old file is left in place (single .1 generation). +func (l *Logger) rotateLocked() { + _ = l.f.Close() + _ = os.Rename(l.path, l.path+".1") + f, err := os.OpenFile(l.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + if err != nil { + l.f = nil + return + } + l.f = f +} + +// Close flushes and closes the log file. Nil-safe. +func (l *Logger) Close() error { + if l == nil { + return nil + } + l.mu.Lock() + defer l.mu.Unlock() + if l.f == nil { + return nil + } + err := l.f.Close() + l.f = nil + return err +} + +// Tail returns the last n entries, oldest first. Nil-safe (returns nil). +func (l *Logger) Tail(n int) ([]Entry, error) { + if l == nil || n <= 0 { + return nil, nil + } + l.mu.Lock() + defer l.mu.Unlock() + b, err := os.ReadFile(l.path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, err + } + var entries []Entry + for _, line := range bytes.Split(bytes.TrimRight(b, "\n"), []byte("\n")) { + var e Entry + if json.Unmarshal(line, &e) == nil { + entries = append(entries, e) + } + } + if len(entries) > n { + entries = entries[len(entries)-n:] + } + return entries, nil +} diff --git a/core/audit/audit_test.go b/core/audit/audit_test.go new file mode 100644 index 0000000..d893c2d --- /dev/null +++ b/core/audit/audit_test.go @@ -0,0 +1,42 @@ +package audit + +import ( + "path/filepath" + "testing" +) + +func TestLoggerAppendTail(t *testing.T) { + dir := t.TempDir() + l, err := Open(filepath.Join(dir, "data")) + if err != nil { + t.Fatal(err) + } + defer l.Close() + + if err := l.Log("1.2.3.4", "delete", "bafyabc", nil); err != nil { + t.Fatal(err) + } + if err := l.Log("5.6.7.8", "drop_all", "", map[string]string{"keys": "3"}); err != nil { + t.Fatal(err) + } + es, err := l.Tail(10) + if err != nil { + t.Fatal(err) + } + if len(es) != 2 || es[0].Action != "delete" || es[1].Action != "drop_all" { + t.Fatalf("Tail = %+v", es) + } + if len(es) == 0 && es != nil { + t.Fatal("unreachable") + } +} + +func TestLoggerNilSafe(t *testing.T) { + var l *Logger + if err := l.Log("x", "y", "z", nil); err != nil { + t.Fatal(err) + } + if _, err := l.Tail(5); err != nil { + t.Fatal(err) + } +} diff --git a/gateway/explorer/audit.go b/gateway/explorer/audit.go new file mode 100644 index 0000000..97840c3 --- /dev/null +++ b/gateway/explorer/audit.go @@ -0,0 +1,23 @@ +package explorer + +import ( + "log/slog" + "net" + "net/http" +) + +// audit records a destructive admin operation in the node's audit +// trail (finding.txt XC-008). Best-effort: failures are logged, never +// surfaced to the requester. +func (e *Explorer) audit(r *http.Request, action, target string, detail map[string]string) { + if e.cfg.Audit == nil { + return + } + actor, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil || actor == "" { + actor = r.RemoteAddr + } + if err := e.cfg.Audit.Log(actor, action, target, detail); err != nil { + slog.Error("explorer: audit write failed", "action", action, "err", err) + } +} diff --git a/gateway/explorer/explorer.go b/gateway/explorer/explorer.go index c4d0891..d974db1 100644 --- a/gateway/explorer/explorer.go +++ b/gateway/explorer/explorer.go @@ -33,6 +33,7 @@ import ( "github.com/gorilla/websocket" "github.com/nnlgsakib/membuss/config" + "github.com/nnlgsakib/membuss/core/audit" "github.com/nnlgsakib/membuss/core/descriptor" "github.com/nnlgsakib/membuss/core/memedge" "github.com/nnlgsakib/membuss/core/mid" @@ -333,6 +334,9 @@ type MemFSEntry struct { type Config struct { // Backend serves the data. Required. Backend Backend + // Audit, if non-nil, records destructive admin operations + // (delete / flash / keyring rm) in /logs/audit.jsonl. + Audit *audit.Logger // ReadTimeout is the per-request read timeout. ReadTimeout time.Duration // WriteTimeout is the per-request write timeout. @@ -766,16 +770,16 @@ func (e *Explorer) handleIndex(w http.ResponseWriter, r *http.Request) { } type midData struct { - Title string - MID string - NotFound bool - MemFSType string - MemFSEntries []MemFSEntry - SymlinkTarget string - Size uint64 - Blocks uint64 - Sealed bool - Codec uint64 + Title string + MID string + NotFound bool + MemFSType string + MemFSEntries []MemFSEntry + SymlinkTarget string + Size uint64 + Blocks uint64 + Sealed bool + Codec uint64 ContentType string DataShards int ParityShards int @@ -785,11 +789,11 @@ type midData struct { ShardMIDs []string Health string HealthLabel string - Providers []string - Name string - MimeType string - Sealers int - AnchorSealers int + Providers []string + Name string + MimeType string + Sealers int + AnchorSealers int // ResolveStatus reports what the explorer's // background fetch attempt did when the MID was // not local. The four interesting values are @@ -824,11 +828,11 @@ func (e *Explorer) handleMID(w http.ResponseWriter, r *http.Request) { present := info.Present size, blocks, sealed, codec := info.Size, info.Blocks, info.Sealed, info.Codec data := midData{ - Title: "MID " + midStr, - MID: midStr, - NotFound: !present, - Name: info.Name, - MimeType: info.MimeType, + Title: "MID " + midStr, + MID: midStr, + NotFound: !present, + Name: info.Name, + MimeType: info.MimeType, Sealers: info.Sealers, AnchorSealers: info.AnchorSealers, } @@ -1087,6 +1091,7 @@ func (e *Explorer) handleDelete(w http.ResponseWriter, r *http.Request) { http.Error(w, "delete: "+err.Error(), http.StatusInternalServerError) return } + e.audit(r, "delete", midStr, nil) http.Redirect(w, r, "/explorer/", http.StatusSeeOther) } @@ -1096,6 +1101,7 @@ func (e *Explorer) handleNodeFlash(w http.ResponseWriter, r *http.Request) { http.Error(w, "flashnode failed: "+err.Error(), http.StatusInternalServerError) return } + e.audit(r, "drop_all", "", nil) if r.URL.Query().Get("format") == "json" || strings.Contains(r.Header.Get("Accept"), "application/json") { w.Header().Set("Content-Type", "application/json") w.Write([]byte(`{"status":"ok","message":"flashnode complete"}`)) @@ -1104,7 +1110,6 @@ func (e *Explorer) handleNodeFlash(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/explorer/node", http.StatusSeeOther) } - type peersData struct { Title string PeerCount int @@ -1216,7 +1221,7 @@ func (e *Explorer) handleNode(w http.ResponseWriter, r *http.Request) { sealed, _ := b.SealedMIDs(ctx) keys, _ := b.KeyringKeys(ctx) e.render(w, r, "node.html", map[string]any{ - "Title": "Node", + "Title": "Node", "NodeInfo": nodeInfoView{ PeerID: b.LocalPeerID(ctx), Addrs: b.LocalAddrs(ctx), @@ -1628,17 +1633,17 @@ func humanBytes(n any) string { } type liveStats struct { - PeerCount int `json:"peerCount"` - StoreBytes uint64 `json:"storeBytes"` - SealedCount int `json:"sealedCount"` - BlockCount uint64 `json:"blockCount"` - Uptime int64 `json:"uptime"` - BandwidthIn float64 `json:"bandwidthIn"` - BandwidthOut float64 `json:"bandwidthOut"` - TotalBytesIn int64 `json:"totalBytesIn"` - TotalBytesOut int64 `json:"totalBytesOut"` - NodeInfo nodeInfoView `json:"nodeInfo"` - SealedList []sealedMIDView `json:"sealedList"` + PeerCount int `json:"peerCount"` + StoreBytes uint64 `json:"storeBytes"` + SealedCount int `json:"sealedCount"` + BlockCount uint64 `json:"blockCount"` + Uptime int64 `json:"uptime"` + BandwidthIn float64 `json:"bandwidthIn"` + BandwidthOut float64 `json:"bandwidthOut"` + TotalBytesIn int64 `json:"totalBytesIn"` + TotalBytesOut int64 `json:"totalBytesOut"` + NodeInfo nodeInfoView `json:"nodeInfo"` + SealedList []sealedMIDView `json:"sealedList"` } var upgrader = websocket.Upgrader{ @@ -1792,6 +1797,7 @@ func (e *Explorer) handleKeyringRm(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + e.audit(r, "key_delete", name, nil) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(map[string]any{"status": "ok", "deleted": name}) From a678e138856a935b831761a163fa5a252aa04b9f Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:26:24 +0600 Subject: [PATCH 12/16] feat(observability): repair metrics + Prometheus dashboards/alerts (XC-009) membuss_erasure_repair_* metrics recorded via RepairWorker.WithMetrics. deploy/ ships two Grafana dashboards (node overview, Mem-Gate) and six alert rules (DHT failure ratio, isolated node, unrecoverable MIDs, gateway 5xx, cache hit ratio, rate-limit drops) with scrape-config README. Note: RepairWorker is not yet started by the daemon (XC-011); repair panels read zero until wired. Co-Authored-By: Claude --- core/erasure/repair.go | 21 +++++- deploy/README.md | 55 ++++++++++++++ deploy/grafana/membuss-gateway.json | 59 +++++++++++++++ deploy/grafana/membuss-node.json | 95 ++++++++++++++++++++++++ deploy/prometheus/rules.yml | 63 ++++++++++++++++ obs/metrics/metrics.go | 107 ++++++++++++++++++++++------ 6 files changed, 377 insertions(+), 23 deletions(-) create mode 100644 deploy/README.md create mode 100644 deploy/grafana/membuss-gateway.json create mode 100644 deploy/grafana/membuss-node.json create mode 100644 deploy/prometheus/rules.yml diff --git a/core/erasure/repair.go b/core/erasure/repair.go index 80e354b..a895657 100644 --- a/core/erasure/repair.go +++ b/core/erasure/repair.go @@ -10,14 +10,15 @@ import ( "github.com/nnlgsakib/membuss/core/mid" "github.com/nnlgsakib/membuss/core/store" + "github.com/nnlgsakib/membuss/obs/metrics" ) // RepairStats tracks background repair worker progress. type RepairStats struct { - AuditedMIDs int - DegradedMIDs int + AuditedMIDs int + DegradedMIDs int RepairedShards int - Unrecoverable int + Unrecoverable int } // RepairMID inspects the erasure shards of a MID and reconstructs missing shards. @@ -110,6 +111,7 @@ type RepairWorker struct { interval time.Duration mu sync.RWMutex stats RepairStats + metrics *metrics.Metrics } // NewRepairWorker creates a new RepairWorker. @@ -123,6 +125,13 @@ func NewRepairWorker(s store.Blockstore, interval time.Duration) *RepairWorker { } } +// WithMetrics attaches the Prometheus handle (XC-009). Returns the +// worker so daemon wiring can chain: NewRepairWorker(...).WithMetrics(m). +func (w *RepairWorker) WithMetrics(m *metrics.Metrics) *RepairWorker { + w.metrics = m + return w +} + // Run executes the background repair loop until ctx is canceled. func (w *RepairWorker) Run(ctx context.Context) { ticker := time.NewTicker(w.interval) @@ -193,6 +202,12 @@ func (w *RepairWorker) AuditAndRepair(ctx context.Context) RepairStats { w.stats.Unrecoverable += stats.Unrecoverable w.mu.Unlock() + if w.metrics != nil { + w.metrics.SetErasureRepairAuditedLastCycle(int64(stats.AuditedMIDs)) + w.metrics.AddErasureRepairShardsRepaired(stats.RepairedShards) + w.metrics.AddErasureRepairUnrecoverable(stats.Unrecoverable) + } + return stats } diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..c03748c --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,55 @@ +# Membuss deploy bundle (XC-009) + +Bundled Prometheus alert rules and Grafana dashboards for the +metrics the daemon already exposes. + +## Contents + +- `grafana/membuss-node.json` — node overview: store size, peers, + DHT provide success, Memex block flow, ingest + transfer latency, + erasure repair activity. +- `grafana/membuss-gateway.json` — Mem-Gate v2: request rate, + 5xx ratio, LRU cache hit ratio, rate-limit drops, SSE streams. +- `prometheus/rules.yml` — 6 alerts (DHT failure ratio, isolated + node, unrecoverable MIDs, gateway 5xx ratio, cache hit ratio, + rate-limit drops). + +## Prometheus + +The node API serves `/metrics` on the API address (default +`127.0.0.1:5001`) when `metrics_enabled: true`. Mem-Gate serves its +own registry on the gateway metrics port (token-gated or +localhost-only). Example scrape config: + +```yaml +scrape_configs: + - job_name: membuss-node + static_configs: + - targets: ["node1:5001"] + + - job_name: membuss-gateway + # gateway metrics are token-gated when metrics_token is set; + # otherwise localhost-only. + authorization: + credentials: "" + static_configs: + - targets: ["node1:8080"] +``` + +## Alert rules + +```yaml +rule_files: + - deploy/prometheus/rules.yml +``` + +## Grafana + +Import both dashboards via **Dashboards > Import** and pick your +Prometheus data source in the `DS_PROMETHEUS` selector. + +## Note on repair metrics + +`membuss_erasure_repair_*` panels stay at zero until the repair +worker is started by the daemon — see finding.txt CORE-F2 / +XC-009 note in finding.txt AUDIT ADDENDUM. diff --git a/deploy/grafana/membuss-gateway.json b/deploy/grafana/membuss-gateway.json new file mode 100644 index 0000000..fa5b8b6 --- /dev/null +++ b/deploy/grafana/membuss-gateway.json @@ -0,0 +1,59 @@ +{ + "title": "Membuss Gateway", + "uid": "membuss-gateway", + "tags": ["membuss"], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": {"from": "now-6h", "to": "now"}, + "templating": {"list": [ + { + "name": "DS_PROMETHEUS", "label": "Data source", + "type": "datasource", "query": "prometheus" + } + ]}, + "panels": [ + { + "id": 1, "type": "stat", "title": "Request rate (5m)", + "gridPos": {"h": 6, "w": 8, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "sum(rate(membuss_gateway_requests_total[5m]))"}] + }, + { + "id": 2, "type": "stat", "title": "Cache hit ratio", + "gridPos": {"h": 6, "w": 8, "x": 8, "y": 0}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "targets": [{"refId": "A", "expr": "sum(membuss_gateway_cache_hits_total) / clamp_min(sum(membuss_gateway_cache_hits_total) + sum(membuss_gateway_cache_misses_total), 1)"}] + }, + { + "id": 3, "type": "stat", "title": "Rate-limit drops (5m rate)", + "gridPos": {"h": 6, "w": 8, "x": 16, "y": 0}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "sum(rate(membuss_gateway_rate_limit_drops_total[5m]))"}] + }, + { + "id": 4, "type": "timeseries", "title": "Requests by status code (per sec)", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "sum by (code) (rate(membuss_gateway_requests_total[5m]))", "legendFormat": "{{code}}"}] + }, + { + "id": 5, "type": "timeseries", "title": "Request latency p95 by method", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, method) (rate(membuss_gateway_request_duration_seconds_bucket[5m])))", "legendFormat": "{{method}}"}] + }, + { + "id": 6, "type": "timeseries", "title": "Active SSE status streams", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 14}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "membuss_gateway_active_sse_streams"}] + } + ] +} diff --git a/deploy/grafana/membuss-node.json b/deploy/grafana/membuss-node.json new file mode 100644 index 0000000..ce248b2 --- /dev/null +++ b/deploy/grafana/membuss-node.json @@ -0,0 +1,95 @@ +{ + "title": "Membuss Node", + "uid": "membuss-node", + "tags": ["membuss"], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": {"from": "now-6h", "to": "now"}, + "templating": {"list": [ + { + "name": "DS_PROMETHEUS", "label": "Data source", + "type": "datasource", "query": "prometheus" + } + ]}, + "panels": [ + { + "id": 1, "type": "stat", "title": "Stored bytes", + "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "membuss_stored_bytes_total"}] + }, + { + "id": 2, "type": "stat", "title": "Stored MIDs", + "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "membuss_stored_mids_total"}] + }, + { + "id": 3, "type": "stat", "title": "Connected peers", + "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "membuss_peers_connected"}] + }, + { + "id": 4, "type": "stat", "title": "GC runs (5m rate)", + "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "targets": [{"refId": "A", "expr": "sum(rate(membuss_gc_runs_total[5m]))"}] + }, + { + "id": 5, "type": "timeseries", "title": "DHT provides / failures (per sec)", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "ops", "custom":{"lineWidth":2,"fillOpacity":10}}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "sum(rate(membuss_dht_provides_total[5m]))", "legendFormat": "provides/s"}, + {"refId": "B", "expr": "sum(rate(membuss_dht_provides_failed_total[5m]))", "legendFormat": "failures/s"} + ] + }, + { + "id": 6, "type": "timeseries", "title": "Memex block flow (per sec)", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "sum(rate(membuss_memex_blocks_sent_total[5m]))", "legendFormat": "sent/s"}, + {"refId": "B", "expr": "sum(rate(membuss_memex_blocks_received_total[5m])", "legendFormat": "received/s"} + ] + }, + { + "id": 7, "type": "timeseries", "title": "Memex transfer latency p50/p95", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.5, sum by (le) (rate(membuss_memex_transfer_duration_seconds_bucket[5m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(membuss_memex_transfer_duration_seconds_bucket[5m])))", "legendFormat": "p95"} + ] + }, + { + "id": 8, "type": "timeseries", "title": "Add ingest latency p95", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(membuss_add_request_duration_seconds_bucket[5m])))", "legendFormat": "p95"} + ] + }, + { + "id": 9, "type": "timeseries", "title": "Erasure repair activity", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 22}, + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "membuss_erasure_repair_audited_last_cycle", "legendFormat": "MIDs audited (last cycle)"}, + {"refId": "B", "expr": "sum(rate(membuss_erasure_repair_shards_repaired_total[5m]))", "legendFormat": "shards repaired/s"}, + {"refId": "C", "expr": "sum(increase(membuss_erasure_repair_unrecoverable_total[1h]))", "legendFormat": "unrecoverable MIDs (1h)"} + ] + } + ] +} diff --git a/deploy/prometheus/rules.yml b/deploy/prometheus/rules.yml new file mode 100644 index 0000000..c9cfd53 --- /dev/null +++ b/deploy/prometheus/rules.yml @@ -0,0 +1,63 @@ +# Prometheus alert rules for Membuss (finding.txt XC-009). +# Load via: rule_files: ["deploy/prometheus/rules.yml"] +groups: + - name: membuss-node + rules: + - alert: MembussDHTProvideFailureRatioHigh + expr: > + sum(rate(membuss_dht_provides_failed_total[15m])) + / clamp_min(sum(rate(membuss_dht_provides_total[15m])), 1e-9) > 0.2 + for: 15m + labels: + severity: warning + annotations: + summary: "DHT provide failure ratio above 20% on {{ $labels.job }}" + description: "Provider announcements are failing; check connectivity, bootstrap peers, and DHT routing table health." + - alert: MembussPeersDisconnected + expr: membuss_peers_connected < 1 + for: 30m + labels: + severity: warning + annotations: + summary: "Membuss node has no connected peers for 30m" + description: "Node is isolated from the swarm; content exchange and provides will fail." + - alert: MembussRepairUnrecoverableMIDs + expr: increase(membuss_erasure_repair_unrecoverable_total[1h]) > 0 + for: 0m + labels: + severity: critical + annotations: + summary: "Erasure repair declared MIDs unrecoverable on {{ $labels.job }}" + description: "Too many RS shards missing to reconstruct; data loss risk. Re-seed from a healthy source." + - name: membuss-gateway + rules: + - alert: MembussGateway5xxRatioHigh + expr: > + sum(rate(membuss_gateway_requests_total{code=~"5.."}[10m])) + / clamp_min(sum(rate(membuss_gateway_requests_total[10m])), 1e-9) > 0.05 + for: 10m + labels: + severity: critical + annotations: + summary: "Mem-Gate 5xx ratio above 5% for 10m" + description: "Gateway error budget burning; inspect backend store and upstream fetch errors." + - alert: MembussGatewayCacheHitRatioLow + expr: > + sum(rate(membuss_gateway_requests_total[10m])) > 1 + and (sum(membuss_gateway_cache_hits_total) + / clamp_min(sum(membuss_gateway_cache_hits_total) + + sum(membuss_gateway_cache_misses_total), 1)) < 0.2 + for: 30m + labels: + severity: info + annotations: + summary: "Gateway LRU cache hit ratio below 20% under load" + description: "Consider raising MaxCacheBytes or checking for one-shot large-object traffic." + - alert: MembussGatewayRateLimitDrops + expr: increase(membuss_gateway_rate_limit_drops_total[10m]) > 100 + for: 0m + labels: + severity: info + annotations: + summary: "More than 100 gateway requests dropped by rate limiting in 10m" + description: "Either abuse or RateLimitPerMin set too low for legitimate traffic." diff --git a/obs/metrics/metrics.go b/obs/metrics/metrics.go index 8b9ee88..c5bd0cd 100644 --- a/obs/metrics/metrics.go +++ b/obs/metrics/metrics.go @@ -28,16 +28,21 @@ type Metrics struct { noop bool registry *prometheus.Registry - storedMIDs prometheus.Gauge - storedBytes prometheus.Gauge - peersConnected prometheus.Gauge - dhtProvides prometheus.Counter - dhtProvidesFailed prometheus.Counter - memexBlocksSent prometheus.Counter - memexBlocksReceived prometheus.Counter - gcRuns prometheus.Counter + storedMIDs prometheus.Gauge + storedBytes prometheus.Gauge + peersConnected prometheus.Gauge + dhtProvides prometheus.Counter + dhtProvidesFailed prometheus.Counter + memexBlocksSent prometheus.Counter + memexBlocksReceived prometheus.Counter + gcRuns prometheus.Counter memexTransferDuration prometheus.Histogram - addRequestDuration prometheus.Histogram + addRequestDuration prometheus.Histogram + + // Erasure repair observability (XC-009). + erasureRepairAuditedLastCycle prometheus.Gauge + erasureRepairShardsRepaired prometheus.Counter + erasureRepairUnrecoverable prometheus.Counter } // noopGuard returns true when the Metrics value is either nil @@ -93,12 +98,27 @@ func New() *Metrics { Help: "Time spent ingesting a new piece of content (chunking + DAG + store).", Buckets: prometheus.DefBuckets, }) + m.erasureRepairAuditedLastCycle = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "membuss_erasure_repair_audited_last_cycle", + Help: "MIDs audited by the most recent erasure repair cycle (0 = worker idle/not running).", + }) + m.erasureRepairShardsRepaired = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "membuss_erasure_repair_shards_repaired_total", + Help: "Cumulative RS shards reconstructed by background repair.", + }) + m.erasureRepairUnrecoverable = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "membuss_erasure_repair_unrecoverable_total", + Help: "Cumulative MIDs declared unrecoverable by repair audit.", + }) reg.MustRegister( m.storedMIDs, m.storedBytes, m.peersConnected, m.dhtProvides, m.dhtProvidesFailed, m.memexBlocksSent, m.memexBlocksReceived, m.gcRuns, m.memexTransferDuration, m.addRequestDuration, + m.erasureRepairAuditedLastCycle, + m.erasureRepairShardsRepaired, + m.erasureRepairUnrecoverable, ) // Standard process collectors give operators free Go runtime // and process metrics. @@ -113,64 +133,109 @@ func Noop() *Metrics { return &Metrics{noop: true} } // SetStoredMIDs updates the gauge of locally stored MIDs. func (m *Metrics) SetStoredMIDs(n int64) { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.storedMIDs.Set(float64(n)) } // SetStoredBytes updates the gauge of locally stored bytes. func (m *Metrics) SetStoredBytes(n uint64) { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.storedBytes.Set(float64(n)) } // SetPeersConnected updates the gauge of currently connected peers. func (m *Metrics) SetPeersConnected(n int) { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.peersConnected.Set(float64(n)) } // IncDHTProvide records one DHT provider announcement. func (m *Metrics) IncDHTProvide() { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.dhtProvides.Inc() } // IncDHTProvideFailed records one failed DHT provider announcement. func (m *Metrics) IncDHTProvideFailed() { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.dhtProvidesFailed.Inc() } // IncMemexBlocksSent records n blocks pushed outbound via Memex. func (m *Metrics) IncMemexBlocksSent(n int) { - if m.noopGuard() || n <= 0 { return } + if m.noopGuard() || n <= 0 { + return + } m.memexBlocksSent.Add(float64(n)) } // IncMemexBlocksReceived records n blocks received inbound via Memex. func (m *Metrics) IncMemexBlocksReceived(n int) { - if m.noopGuard() || n <= 0 { return } + if m.noopGuard() || n <= 0 { + return + } m.memexBlocksReceived.Add(float64(n)) } // IncGCRuns records one garbage-collection run. func (m *Metrics) IncGCRuns() { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.gcRuns.Inc() } // ObserveMemexTransfer records a single Memex transfer duration. func (m *Metrics) ObserveMemexTransfer(seconds float64) { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.memexTransferDuration.Observe(seconds) } // ObserveAddRequest records a single Add (ingest) request duration. func (m *Metrics) ObserveAddRequest(seconds float64) { - if m.noopGuard() { return } + if m.noopGuard() { + return + } m.addRequestDuration.Observe(seconds) } +// SetErasureRepairAuditedLastCycle records how many MIDs the most +// recent repair cycle audited (XC-009). +func (m *Metrics) SetErasureRepairAuditedLastCycle(n int64) { + if m.noopGuard() { + return + } + m.erasureRepairAuditedLastCycle.Set(float64(n)) +} + +// AddErasureRepairShardsRepaired records n reconstructed RS shards. +func (m *Metrics) AddErasureRepairShardsRepaired(n int) { + if m.noopGuard() { + return + } + m.erasureRepairShardsRepaired.Add(float64(n)) +} + +// AddErasureRepairUnrecoverable records n unrecoverable MIDs. +func (m *Metrics) AddErasureRepairUnrecoverable(n int) { + if m.noopGuard() || n <= 0 { + return + } + m.erasureRepairUnrecoverable.Add(float64(n)) +} + // Handler returns an http.Handler that serves the registry in the // Prometheus text format. A nil receiver returns a handler that // responds with 503 Service Unavailable. @@ -186,6 +251,8 @@ func (m *Metrics) Handler() http.Handler { // Registry returns the underlying registry, for tests that want to // gather values directly. nil for noop metrics. func (m *Metrics) Registry() *prometheus.Registry { - if m == nil { return nil } + if m == nil { + return nil + } return m.registry } From 811a685093cb9f3b5562af930d448f1dc96a146a Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:26:41 +0600 Subject: [PATCH 13/16] feat(explorer-web): a11y pass + i18n scaffolding (XC-010) Skip link + main landmark, aria-current nav, labeled search inputs/button, SwarmMap cluster labels, WASM upload input label. New src/lib/i18n: t() store + en dictionary; shared nav wired through it. Route-level strings migrate incrementally. Co-Authored-By: Claude --- .../src/lib/components/SwarmMap.svelte | 1 + explorer-web/src/lib/i18n/index.ts | 18 ++++++++++ explorer-web/src/lib/i18n/locales/en.ts | 14 ++++++++ explorer-web/src/routes/+layout.svelte | 36 ++++++++++++------- explorer-web/src/routes/edge/+page.svelte | 1 + 5 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 explorer-web/src/lib/i18n/index.ts create mode 100644 explorer-web/src/lib/i18n/locales/en.ts diff --git a/explorer-web/src/lib/components/SwarmMap.svelte b/explorer-web/src/lib/components/SwarmMap.svelte index 847776d..acc2853 100644 --- a/explorer-web/src/lib/components/SwarmMap.svelte +++ b/explorer-web/src/lib/components/SwarmMap.svelte @@ -199,6 +199,7 @@ enter(c)} onmouseleave={scheduleClose} diff --git a/explorer-web/src/lib/i18n/index.ts b/explorer-web/src/lib/i18n/index.ts new file mode 100644 index 0000000..9c7d222 --- /dev/null +++ b/explorer-web/src/lib/i18n/index.ts @@ -0,0 +1,18 @@ +// Minimal i18n layer (finding.txt XC-010). Synchronous dictionary +// lookup backed by svelte stores: `t` is a derived store returning a +// translate function, so components use it as {$t('nav.status')}. +// +// Adding a locale: create locales/.ts exporting the same key +// set, then register it below. Missing keys fall back to English, +// then to the raw key. +import { derived, writable } from 'svelte/store'; +import { en } from './locales/en'; + +const dictionaries: Record> = { en }; + +export type Locale = keyof typeof dictionaries; +export const locale = writable('en'); + +export const t = derived(locale, (l) => (key: string): string => + dictionaries[l]?.[key] ?? dictionaries.en[key] ?? key +); diff --git a/explorer-web/src/lib/i18n/locales/en.ts b/explorer-web/src/lib/i18n/locales/en.ts new file mode 100644 index 0000000..3df3970 --- /dev/null +++ b/explorer-web/src/lib/i18n/locales/en.ts @@ -0,0 +1,14 @@ +// Base English dictionary. Dot-namespaced keys, grouped by surface. +// Route-level strings migrate here incrementally; shared chrome +// (nav, search, footer) is wired through t() first. +export const en = { + // Navigation (shared chrome) + 'nav.status': 'Status', + 'nav.files': 'Files', + 'nav.explore': 'Explore', + 'nav.memns': 'MemNS', + 'nav.edge': 'Edge', + 'nav.peers': 'Peers', + 'nav.tunnel': 'Tunnel', + 'nav.node': 'Node Info' +} as const; diff --git a/explorer-web/src/routes/+layout.svelte b/explorer-web/src/routes/+layout.svelte index 4bf993d..dfb169b 100644 --- a/explorer-web/src/routes/+layout.svelte +++ b/explorer-web/src/routes/+layout.svelte @@ -9,6 +9,7 @@ import Icon from '@iconify/svelte'; import Toasts from '$lib/components/Toasts.svelte'; import UploadWidget from '$lib/components/UploadWidget.svelte'; + import { t } from '$lib/i18n'; let { children } = $props(); @@ -17,14 +18,14 @@ // Ordered by priority, clustered by domain: overview → content → network → system const navItems = [ - { name: 'Status', path: '/', icon: 'ph:gauge-light', group: 'overview' }, - { name: 'Files', path: '/files', icon: 'ph:folder-open-light', group: 'content' }, - { name: 'Explore', path: '/explore', icon: 'ph:git-branch-light', group: 'content' }, - { name: 'MemNS', path: '/memns', icon: 'ph:identification-card-light', group: 'content' }, - { name: 'Edge', path: '/edge', icon: 'ph:lightning-light', group: 'compute' }, - { name: 'Peers', path: '/peers', icon: 'ph:circle-notch-light', group: 'network' }, - { name: 'Tunnel', path: '/tunnel', icon: 'ph:link-light', group: 'network' }, - { name: 'Node Info', path: '/node', icon: 'ph:gear-six-light', group: 'system' } + { key: 'nav.status', path: '/', icon: 'ph:gauge-light', group: 'overview' }, + { key: 'nav.files', path: '/files', icon: 'ph:folder-open-light', group: 'content' }, + { key: 'nav.explore', path: '/explore', icon: 'ph:git-branch-light', group: 'content' }, + { key: 'nav.memns', path: '/memns', icon: 'ph:identification-card-light', group: 'content' }, + { key: 'nav.edge', path: '/edge', icon: 'ph:lightning-light', group: 'compute' }, + { key: 'nav.peers', path: '/peers', icon: 'ph:circle-notch-light', group: 'network' }, + { key: 'nav.tunnel', path: '/tunnel', icon: 'ph:link-light', group: 'network' }, + { key: 'nav.node', path: '/node', icon: 'ph:gear-six-light', group: 'system' } ]; function handleSearch(e: Event) { @@ -89,6 +90,13 @@ -
+
{@render children()}
diff --git a/explorer-web/src/routes/edge/+page.svelte b/explorer-web/src/routes/edge/+page.svelte index 53b1285..a12eb30 100644 --- a/explorer-web/src/routes/edge/+page.svelte +++ b/explorer-web/src/routes/edge/+page.svelte @@ -1102,6 +1102,7 @@ fn main() -> io::Result<()> { { const target = e.target as HTMLInputElement; if (target.files?.[0]) handleWasmFileUpload(target.files[0]); From 9c266c83cf4c67e479d502a487ffb86e35374ac4 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:26:49 +0600 Subject: [PATCH 14/16] chore: strip leading UTF-8 BOMs from Go sources 20 files carried an EF BB BF prefix (legal per spec but unusual; some toolchains reject it). Bytes-only change, no code edits. Co-Authored-By: Claude --- anchor/doc.go | 2 +- anchor/helpers_test.go | 2 +- config/datadir.go | 2 +- config/datadir_test.go | 2 +- config/default.go | 2 +- core/chunk/doc.go | 2 +- core/dag/dag_test.go | 2 +- core/dag/doc.go | 2 +- core/dag/resolve.go | 2 +- core/erasure/doc.go | 2 +- core/mid/doc.go | 2 +- core/shard/doc.go | 2 +- core/shard/hashring_test.go | 2 +- core/store/doc.go | 2 +- core/store/store_test.go | 2 +- net/dht/doc.go | 2 +- net/dht/inproc_host_test.go | 2 +- net/host/doc.go | 2 +- net/host/identity.go | 2 +- net/host/identity_test.go | 2 +- net/host/persistent_identity_test.go | 2 +- net/pex/doc.go | 2 +- net/pex/inproc_host_test.go | 2 +- rpc/proto/doc.go | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/anchor/doc.go b/anchor/doc.go index 197988d..d40236f 100644 --- a/anchor/doc.go +++ b/anchor/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // Anchor Node: full-network sync engine for high availability. package anchor diff --git a/anchor/helpers_test.go b/anchor/helpers_test.go index ca04202..81a7ebb 100644 --- a/anchor/helpers_test.go +++ b/anchor/helpers_test.go @@ -1,4 +1,4 @@ -package anchor +package anchor import ( "crypto/rand" diff --git a/config/datadir.go b/config/datadir.go index 8848d84..1a3feec 100644 --- a/config/datadir.go +++ b/config/datadir.go @@ -1,4 +1,4 @@ -// Phase 16: data-directory resolution and the YAML-with-comments +// Phase 16: data-directory resolution and the YAML-with-comments // serializer used by `membuss init`. // // The data directory is the single place a node keeps its diff --git a/config/datadir_test.go b/config/datadir_test.go index 7e6fe99..e88bc84 100644 --- a/config/datadir_test.go +++ b/config/datadir_test.go @@ -1,4 +1,4 @@ -package config +package config import ( "os" diff --git a/config/default.go b/config/default.go index 4c10859..2952808 100644 --- a/config/default.go +++ b/config/default.go @@ -1,4 +1,4 @@ -package config +package config import ( "os" diff --git a/core/chunk/doc.go b/core/chunk/doc.go index 227aaac..59e488c 100644 --- a/core/chunk/doc.go +++ b/core/chunk/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // Chunking engine: splits content into fixed-size blocks for the Merkle DAG. package chunk diff --git a/core/dag/dag_test.go b/core/dag/dag_test.go index 29bc715..66ea6e5 100644 --- a/core/dag/dag_test.go +++ b/core/dag/dag_test.go @@ -1,4 +1,4 @@ -package dag +package dag import ( "bytes" diff --git a/core/dag/doc.go b/core/dag/doc.go index ff15578..eddb2de 100644 --- a/core/dag/doc.go +++ b/core/dag/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // Merkle DAG builder and resolver. package dag diff --git a/core/dag/resolve.go b/core/dag/resolve.go index dc00036..683d88b 100644 --- a/core/dag/resolve.go +++ b/core/dag/resolve.go @@ -1,4 +1,4 @@ -package dag +package dag import ( "bytes" diff --git a/core/erasure/doc.go b/core/erasure/doc.go index 8f19836..1279506 100644 --- a/core/erasure/doc.go +++ b/core/erasure/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // Reed-Solomon erasure coding over data chunks. package erasure diff --git a/core/mid/doc.go b/core/mid/doc.go index 5bd4441..3901deb 100644 --- a/core/mid/doc.go +++ b/core/mid/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // MID generation, parsing, and validation (multihash with the "mem" codec prefix). package mid diff --git a/core/shard/doc.go b/core/shard/doc.go index 763f0ea..e496c64 100644 --- a/core/shard/doc.go +++ b/core/shard/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // Consistent-hash sharding logic for placing shards across peers. package shard diff --git a/core/shard/hashring_test.go b/core/shard/hashring_test.go index 0c18c14..2b24a0f 100644 --- a/core/shard/hashring_test.go +++ b/core/shard/hashring_test.go @@ -1,4 +1,4 @@ -package shard +package shard import ( "fmt" diff --git a/core/store/doc.go b/core/store/doc.go index 16b2258..f1243a9 100644 --- a/core/store/doc.go +++ b/core/store/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // BadgerDB-backed blockstore (Mem-Store). package store diff --git a/core/store/store_test.go b/core/store/store_test.go index 97e0fe1..edfc465 100644 --- a/core/store/store_test.go +++ b/core/store/store_test.go @@ -1,4 +1,4 @@ -package store +package store import ( "errors" diff --git a/net/dht/doc.go b/net/dht/doc.go index fbcc7d6..b3f8bed 100644 --- a/net/dht/doc.go +++ b/net/dht/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // Mem-DHT: Kademlia-based peer and content discovery. package dht diff --git a/net/dht/inproc_host_test.go b/net/dht/inproc_host_test.go index 11ebd92..8ab2bc9 100644 --- a/net/dht/inproc_host_test.go +++ b/net/dht/inproc_host_test.go @@ -1,4 +1,4 @@ -package dht +package dht import ( "crypto/rand" diff --git a/net/host/doc.go b/net/host/doc.go index 1081564..f27c058 100644 --- a/net/host/doc.go +++ b/net/host/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // libp2p host construction and lifecycle. package host diff --git a/net/host/identity.go b/net/host/identity.go index b8505be..d6e1c42 100644 --- a/net/host/identity.go +++ b/net/host/identity.go @@ -1,4 +1,4 @@ -// Phase 16: identity persistence helpers used by +// Phase 16: identity persistence helpers used by // `membuss-cli init` and (as a fallback) by NewHost. // // The host package has always been able to load or create the diff --git a/net/host/identity_test.go b/net/host/identity_test.go index c7213b8..f2bba4e 100644 --- a/net/host/identity_test.go +++ b/net/host/identity_test.go @@ -1,4 +1,4 @@ -package host +package host import ( "os" diff --git a/net/host/persistent_identity_test.go b/net/host/persistent_identity_test.go index 80d32e6..bf33001 100644 --- a/net/host/persistent_identity_test.go +++ b/net/host/persistent_identity_test.go @@ -1,4 +1,4 @@ -package host +package host import ( "os" diff --git a/net/pex/doc.go b/net/pex/doc.go index 9a6efac..ce47ac5 100644 --- a/net/pex/doc.go +++ b/net/pex/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // PEX: Peer Exchange gossip protocol. package pex diff --git a/net/pex/inproc_host_test.go b/net/pex/inproc_host_test.go index f183f88..8ec93f1 100644 --- a/net/pex/inproc_host_test.go +++ b/net/pex/inproc_host_test.go @@ -1,4 +1,4 @@ -package pex +package pex import ( "crypto/rand" diff --git a/rpc/proto/doc.go b/rpc/proto/doc.go index 3f48d4f..9027567 100644 --- a/rpc/proto/doc.go +++ b/rpc/proto/doc.go @@ -1,4 +1,4 @@ -// Package proto holds the hand-written .proto files that describe the +// Package proto holds the hand-written .proto files that describe the // Membuss wire formats. The generated Go bindings live in /proto and // are produced by `make proto`. package proto From 8af5f20d0c25c810a357efe41f822e10d0f11499 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:27:00 +0600 Subject: [PATCH 15/16] chore(release): bump version to v2.9.4 Co-Authored-By: Claude --- core/version/version.go | 2 +- desktop/frontend/src/App.svelte | 2 +- desktop/frontend/src/components/Dashboard.svelte | 4 ++-- desktop/frontend/src/components/UpdateModal.svelte | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/version/version.go b/core/version/version.go index fdfcefc..ed5bf76 100644 --- a/core/version/version.go +++ b/core/version/version.go @@ -10,7 +10,7 @@ var ( // Version is the semantic version of the application. // Can be overridden at build time via: // -ldflags "-X github.com/nnlgsakib/membuss/core/version.Version=1.0.0" - Version = "2.9.3" + Version = "2.9.4" // GitCommit is the git commit SHA. // Can be overridden at build time via: diff --git a/desktop/frontend/src/App.svelte b/desktop/frontend/src/App.svelte index d1026ad..84ff9dd 100644 --- a/desktop/frontend/src/App.svelte +++ b/desktop/frontend/src/App.svelte @@ -35,7 +35,7 @@
MEMBUSS - {app.config?.installed_version || 'v2.9.3'} + {app.config?.installed_version || 'v2.9.4'}
decentralized content network diff --git a/desktop/frontend/src/components/Dashboard.svelte b/desktop/frontend/src/components/Dashboard.svelte index a9768b7..522fcd9 100644 --- a/desktop/frontend/src/components/Dashboard.svelte +++ b/desktop/frontend/src/components/Dashboard.svelte @@ -168,7 +168,7 @@ Node Daemon is in Standby - {app.config?.installed_version || 'v2.9.3'} + {app.config?.installed_version || 'v2.9.4'} @@ -348,7 +348,7 @@ Local Node Online & Active - {app.nodeStatus.info?.version ? `v${app.nodeStatus.info.version}` : (app.config?.installed_version || 'v2.9.3')} + {app.nodeStatus.info?.version ? `v${app.nodeStatus.info.version}` : (app.config?.installed_version || 'v2.9.4')} diff --git a/desktop/frontend/src/components/UpdateModal.svelte b/desktop/frontend/src/components/UpdateModal.svelte index 938d108..8769503 100644 --- a/desktop/frontend/src/components/UpdateModal.svelte +++ b/desktop/frontend/src/components/UpdateModal.svelte @@ -55,7 +55,7 @@
Installed - {app.updateInfo?.current_version || app.config?.installed_version || 'v2.9.3'} + {app.updateInfo?.current_version || app.config?.installed_version || 'v2.9.4'}
From 7f369d344218f65afddf144b6b0d4f6e893574f2 Mon Sep 17 00:00:00 2001 From: nnlgsakib Date: Sat, 22 Aug 2026 20:27:23 +0600 Subject: [PATCH 16/16] chore(api): strip UTF-8 BOM from doc.go Missed in the BOM sweep commit. Co-Authored-By: Claude --- api/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/doc.go b/api/doc.go index 278116b..9f4399c 100644 --- a/api/doc.go +++ b/api/doc.go @@ -1,4 +1,4 @@ -// Package is part of the Membuss skeleton. +// Package is part of the Membuss skeleton. // // Local Node control API: add, seal, query operations. package api