From 9e95a26f0a89e1fda01a4d0375e1e3e6f368d21a Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Mon, 7 Jul 2025 12:46:08 +0200 Subject: [PATCH 1/3] reprovider implements dht provide interface --- go.mod | 6 ++++-- go.sum | 12 +++++++---- provider/reprovider.go | 46 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 56fdcc892..08c060650 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/libp2p/go-buffer-pool v0.1.0 github.com/libp2p/go-doh-resolver v0.5.0 github.com/libp2p/go-libp2p v0.42.0 - github.com/libp2p/go-libp2p-kad-dht v0.33.1 + github.com/libp2p/go-libp2p-kad-dht v0.33.2-0.20250707091924-e2e28c747ebb github.com/libp2p/go-libp2p-record v0.3.1 github.com/libp2p/go-libp2p-routing-helpers v0.7.5 github.com/libp2p/go-libp2p-testing v0.12.0 @@ -96,6 +96,7 @@ require ( github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/guillaumemichel/reservedpool v0.1.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect @@ -108,7 +109,7 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kbucket v0.7.0 // indirect + github.com/libp2p/go-libp2p-kbucket v0.7.1-0.20250514131430-6bb8b16e312b // indirect github.com/libp2p/go-netroute v0.2.2 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v5 v5.0.1 // indirect @@ -145,6 +146,7 @@ require ( github.com/pion/turn/v4 v4.0.2 // indirect github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/probe-lab/go-libdht v0.2.1-0.20250704053741-31e98600d9ec // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect diff --git a/go.sum b/go.sum index 714ff6af2..d04ada762 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/guillaumemichel/reservedpool v0.1.0 h1:QOG3bsExi+Erk2TBzGan87uqA48Ns27uhhnwgCJup+Y= +github.com/guillaumemichel/reservedpool v0.1.0/go.mod h1:sXSDIaef81TFdAJglsCFCMfgF5E5Z5xK1tFhjDhvbUc= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -226,10 +228,10 @@ github.com/libp2p/go-libp2p v0.42.0 h1:A8foZk+ZEhZTv0Jb++7xUFlrFhBDv4j2Vh/uq4YX+ github.com/libp2p/go-libp2p v0.42.0/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= -github.com/libp2p/go-libp2p-kad-dht v0.33.1 h1:hKFhHMf7WH69LDjaxsJUWOU6qZm71uO47M/a5ijkiP0= -github.com/libp2p/go-libp2p-kad-dht v0.33.1/go.mod h1:CdmNk4VeGJa9EXM9SLNyNVySEvduKvb+5rSC/H4pLAo= -github.com/libp2p/go-libp2p-kbucket v0.7.0 h1:vYDvRjkyJPeWunQXqcW2Z6E93Ywx7fX0jgzb/dGOKCs= -github.com/libp2p/go-libp2p-kbucket v0.7.0/go.mod h1:blOINGIj1yiPYlVEX0Rj9QwEkmVnz3EP8LK1dRKBC6g= +github.com/libp2p/go-libp2p-kad-dht v0.33.2-0.20250707091924-e2e28c747ebb h1:nwODa+HiE8U9VQqPHBTepiMLNpJzl9vMddFHRbwcGbc= +github.com/libp2p/go-libp2p-kad-dht v0.33.2-0.20250707091924-e2e28c747ebb/go.mod h1:fVspLz4QG2bj+4zMSn6Lfigbzwc2Ie7wpMp+IAvRyqQ= +github.com/libp2p/go-libp2p-kbucket v0.7.1-0.20250514131430-6bb8b16e312b h1:qh7OgCwt5AjJohJH+oXKZw6iDeIWKJOZaLXwkxrW0WQ= +github.com/libp2p/go-libp2p-kbucket v0.7.1-0.20250514131430-6bb8b16e312b/go.mod h1:3CofRbwJbTybT8WVM2z/h5dj2FPNar6YLyxUc4Tmv1E= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= @@ -357,6 +359,8 @@ github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4 github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/probe-lab/go-libdht v0.2.1-0.20250704053741-31e98600d9ec h1:SHYUHNzEwackoRJ0yHw8Bx39kQYL2BtJ45qchJfbV0k= +github.com/probe-lab/go-libdht v0.2.1-0.20250704053741-31e98600d9ec/go.mod h1:q+WlGiqs/UIRfdhw9Gmc+fPoAYlOim7VvXTjOI6KJmQ= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= diff --git a/provider/reprovider.go b/provider/reprovider.go index 100aee456..cd823ccc3 100644 --- a/provider/reprovider.go +++ b/provider/reprovider.go @@ -16,7 +16,8 @@ import ( "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" metrics "github.com/ipfs/go-metrics-interface" - "github.com/multiformats/go-multihash" + "github.com/libp2p/go-libp2p-kad-dht/provider" + mh "github.com/multiformats/go-multihash" ) const ( @@ -80,14 +81,17 @@ type reprovider struct { keyPrefix datastore.Key } -var _ System = (*reprovider)(nil) +var ( + _ System = (*reprovider)(nil) + _ provider.Provider = (*reprovider)(nil) +) type Provide interface { Provide(context.Context, cid.Cid, bool) error } type ProvideMany interface { - ProvideMany(ctx context.Context, keys []multihash.Multihash) error + ProvideMany(ctx context.Context, keys []mh.Multihash) error } type Ready interface { @@ -445,7 +449,7 @@ func (s *reprovider) Reprovide(ctx context.Context) error { return errors.New("failed to reprovide: shutting down") } - keys := make([]multihash.Multihash, 0, len(cids)) + keys := make([]mh.Multihash, 0, len(cids)) for c := range cids { // hash security if err := verifcid.ValidateCid(s.allowlist, c); err != nil { @@ -544,7 +548,7 @@ func (s *reprovider) Stat() (ReproviderStats, error) { }, nil } -func doProvideMany(ctx context.Context, r Provide, keys []multihash.Multihash) error { +func doProvideMany(ctx context.Context, r Provide, keys []mh.Multihash) error { if many, ok := r.(ProvideMany); ok { return many.ProvideMany(ctx, keys) } @@ -557,3 +561,35 @@ func doProvideMany(ctx context.Context, r Provide, keys []multihash.Multihash) e } return nil } + +// StartProviding doesn't keep track of which keys have been provided so far. +// It simply calls InstantProvide to provide the given keys to the network, and +// returns instantly. +func (r *reprovider) StartProviding(keys ...mh.Multihash) { + go r.InstantProvide(context.Background(), keys...) +} + +// StopProviding is a no op, since reprovider isn't tracking the keys to be +// reprovided over time. +func (r *reprovider) StopProviding(keys ...mh.Multihash) {} + +// InstantProvide provides the given keys to the network without waiting. +// +// If an error is returned by the Provide operation, don't try to provide the +// remaining keys, and return the error. +func (r *reprovider) InstantProvide(ctx context.Context, keys ...mh.Multihash) error { + for _, k := range keys { + err := r.Provide(ctx, cid.NewCidV1(cid.Raw, k), true) + if err != nil { + return err + } + } + return nil +} + +// ForceProvide is an alias for InstantProvide, it provides the given keys to +// the network, but doesn't track which keys should be reprovided since +// reprovider doesn't hold such a state. +func (r *reprovider) ForceProvide(ctx context.Context, keys ...mh.Multihash) error { + return r.InstantProvide(ctx, keys...) +} From 771b3e66b0a2d45db3b0cf6333aaf138e23c0a15 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Tue, 8 Jul 2025 09:55:50 +0200 Subject: [PATCH 2/3] add Provider constructor --- provider/reprovider.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/provider/reprovider.go b/provider/reprovider.go index cd823ccc3..77d6d48b2 100644 --- a/provider/reprovider.go +++ b/provider/reprovider.go @@ -593,3 +593,12 @@ func (r *reprovider) InstantProvide(ctx context.Context, keys ...mh.Multihash) e func (r *reprovider) ForceProvide(ctx context.Context, keys ...mh.Multihash) error { return r.InstantProvide(ctx, keys...) } + +// NewProvider creates a new [provider.Provider] by casting the result of New() +func NewProvider(ds datastore.Batching, opts ...Option) (provider.Provider, error) { + system, err := New(ds, opts...) + if err != nil { + return nil, err + } + return system.(*reprovider), nil +} From d48ea9a628f4934162f37caa785a7f64cd54eee3 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Tue, 8 Jul 2025 09:57:38 +0200 Subject: [PATCH 3/3] examples go mod tidy --- examples/go.mod | 4 ++-- examples/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/go.mod b/examples/go.mod index 49c73418b..a1c9181cd 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -74,8 +74,8 @@ require ( github.com/libp2p/go-doh-resolver v0.5.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.33.1 // indirect - github.com/libp2p/go-libp2p-kbucket v0.7.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.33.2-0.20250707091924-e2e28c747ebb // indirect + github.com/libp2p/go-libp2p-kbucket v0.7.1-0.20250514131430-6bb8b16e312b // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect diff --git a/examples/go.sum b/examples/go.sum index a57fcd982..6d905e8b8 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -225,10 +225,10 @@ github.com/libp2p/go-libp2p v0.42.0 h1:A8foZk+ZEhZTv0Jb++7xUFlrFhBDv4j2Vh/uq4YX+ github.com/libp2p/go-libp2p v0.42.0/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= -github.com/libp2p/go-libp2p-kad-dht v0.33.1 h1:hKFhHMf7WH69LDjaxsJUWOU6qZm71uO47M/a5ijkiP0= -github.com/libp2p/go-libp2p-kad-dht v0.33.1/go.mod h1:CdmNk4VeGJa9EXM9SLNyNVySEvduKvb+5rSC/H4pLAo= -github.com/libp2p/go-libp2p-kbucket v0.7.0 h1:vYDvRjkyJPeWunQXqcW2Z6E93Ywx7fX0jgzb/dGOKCs= -github.com/libp2p/go-libp2p-kbucket v0.7.0/go.mod h1:blOINGIj1yiPYlVEX0Rj9QwEkmVnz3EP8LK1dRKBC6g= +github.com/libp2p/go-libp2p-kad-dht v0.33.2-0.20250707091924-e2e28c747ebb h1:nwODa+HiE8U9VQqPHBTepiMLNpJzl9vMddFHRbwcGbc= +github.com/libp2p/go-libp2p-kad-dht v0.33.2-0.20250707091924-e2e28c747ebb/go.mod h1:fVspLz4QG2bj+4zMSn6Lfigbzwc2Ie7wpMp+IAvRyqQ= +github.com/libp2p/go-libp2p-kbucket v0.7.1-0.20250514131430-6bb8b16e312b h1:qh7OgCwt5AjJohJH+oXKZw6iDeIWKJOZaLXwkxrW0WQ= +github.com/libp2p/go-libp2p-kbucket v0.7.1-0.20250514131430-6bb8b16e312b/go.mod h1:3CofRbwJbTybT8WVM2z/h5dj2FPNar6YLyxUc4Tmv1E= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI=