diff --git a/cmd/lassie/daemon.go b/cmd/lassie/daemon.go index c84f686e..effa882f 100644 --- a/cmd/lassie/daemon.go +++ b/cmd/lassie/daemon.go @@ -7,9 +7,6 @@ import ( "github.com/filecoin-project/lassie/pkg/aggregateeventrecorder" "github.com/filecoin-project/lassie/pkg/lassie" httpserver "github.com/filecoin-project/lassie/pkg/server/http" - "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p/config" - "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/urfave/cli/v2" ) @@ -38,44 +35,16 @@ var daemonFlags = []cli.Flag{ DefaultText: "no limit", EnvVars: []string{"LASSIE_MAX_BLOCKS_PER_REQUEST"}, }, - &cli.IntFlag{ - Name: "libp2p-conns-lowwater", - Aliases: []string{"lw"}, - Usage: "lower limit of libp2p connections", - Value: 0, - DefaultText: "libp2p default", - EnvVars: []string{"LASSIE_LIBP2P_CONNECTIONS_LOWWATER"}, - }, - &cli.IntFlag{ - Name: "libp2p-conns-highwater", - Aliases: []string{"hw"}, - Usage: "upper limit of libp2p connections", - Value: 0, - DefaultText: "libp2p default", - EnvVars: []string{"LASSIE_LIBP2P_CONNECTIONS_HIGHWATER"}, - }, - &cli.UintFlag{ - Name: "concurrent-sp-retrievals", - Aliases: []string{"cr"}, - Usage: "max number of simultaneous SP retrievals", - Value: 0, - DefaultText: "no limit", - EnvVars: []string{"LASSIE_CONCURRENT_SP_RETRIEVALS"}, - }, - FlagIPNIEndpoint, + FlagDelegatedRoutingEndpoint, FlagEventRecorderAuth, FlagEventRecorderInstanceId, FlagEventRecorderUrl, FlagVerbose, FlagVeryVerbose, - FlagProtocols, FlagAllowProviders, FlagExcludeProviders, FlagTempDir, - FlagBitswapConcurrency, - FlagBitswapConcurrencyPerRetrieval, FlagGlobalTimeout, - FlagProviderTimeout, &cli.StringFlag{ Name: "access-token", Usage: "require HTTP clients to authorize using Bearer scheme and given access token", @@ -96,26 +65,9 @@ var daemonCmd = &cli.Command{ // the cli context into the appropriate config objects and then calls the // daemonRun function. func daemonAction(cctx *cli.Context) error { - // lassie config - libp2pLowWater := cctx.Int("libp2p-conns-lowwater") - libp2pHighWater := cctx.Int("libp2p-conns-highwater") - concurrentSPRetrievals := cctx.Uint("concurrent-sp-retrievals") lassieOpts := []lassie.LassieOption{} - if concurrentSPRetrievals > 0 { - lassieOpts = append(lassieOpts, lassie.WithConcurrentSPRetrievals(concurrentSPRetrievals)) - } - - libp2pOpts := []config.Option{} - if libp2pHighWater != 0 || libp2pLowWater != 0 { - connManager, err := connmgr.NewConnManager(libp2pLowWater, libp2pHighWater) - if err != nil { - return cli.Exit(err, 1) - } - libp2pOpts = append(libp2pOpts, libp2p.ConnectionManager(connManager)) - } - - lassieCfg, err := buildLassieConfigFromCLIContext(cctx, lassieOpts, libp2pOpts) + lassieCfg, err := buildLassieConfigFromCLIContext(cctx, lassieOpts) if err != nil { return err } @@ -167,17 +119,17 @@ func defaultDaemonRun( httpServerCfg httpserver.HttpServerConfig, eventRecorderCfg *aggregateeventrecorder.EventRecorderConfig, ) error { - lassie, err := lassie.NewLassieWithConfig(ctx, lassieCfg) + s, err := lassie.NewLassieWithConfig(ctx, lassieCfg) if err != nil { - return nil + return err } // create and subscribe an event recorder API if an endpoint URL is set if eventRecorderCfg.EndpointURL != "" { - setupLassieEventRecorder(ctx, eventRecorderCfg, lassie) + setupLassieEventRecorder(ctx, eventRecorderCfg, s) } - httpServer, err := httpserver.NewHttpServer(ctx, lassie, httpServerCfg) + httpServer, err := httpserver.NewHttpServer(ctx, s, httpServerCfg) if err != nil { logger.Errorw("failed to create http server", "err", err) return err diff --git a/cmd/lassie/daemon_test.go b/cmd/lassie/daemon_test.go deleted file mode 100644 index d4660357..00000000 --- a/cmd/lassie/daemon_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package main - -import ( - "context" - "testing" - "time" - - a "github.com/filecoin-project/lassie/pkg/aggregateeventrecorder" - "github.com/filecoin-project/lassie/pkg/indexerlookup" - l "github.com/filecoin-project/lassie/pkg/lassie" - h "github.com/filecoin-project/lassie/pkg/server/http" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/p2p/net/connmgr" - "github.com/multiformats/go-multicodec" - "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" -) - -func TestDaemonCommandFlags(t *testing.T) { - tests := []struct { - name string - args []string - shouldError bool - assert daemonRunFunc - }{ - { - name: "with default args", - args: []string{"daemon"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - // lassie config - require.Equal(t, nil, lCfg.Source) - require.NotNil(t, lCfg.Host, "host should not be nil") - require.Equal(t, 20*time.Second, lCfg.ProviderTimeout) - require.Equal(t, uint(0), lCfg.ConcurrentSPRetrievals) - require.Equal(t, 0*time.Second, lCfg.GlobalTimeout) - require.Equal(t, 0, len(lCfg.Libp2pOptions)) - require.Equal(t, 0, len(lCfg.Protocols)) - require.Equal(t, 0, len(lCfg.ProviderBlockList)) - require.Equal(t, 0, len(lCfg.ProviderAllowList)) - require.Equal(t, 32, lCfg.BitswapConcurrency) - require.Equal(t, 12, lCfg.BitswapConcurrencyPerRetrieval) - - // http server config - require.Equal(t, "127.0.0.1", hCfg.Address) - require.Equal(t, uint(0), hCfg.Port) - require.Equal(t, uint64(0), hCfg.MaxBlocksPerRequest) - require.Equal(t, "", hCfg.AccessToken) - - // event recorder config - require.Equal(t, "", erCfg.EndpointURL) - require.Equal(t, "", erCfg.EndpointAuthorization) - require.Equal(t, "", erCfg.InstanceID) - return nil - }, - }, - { - name: "with libp2p low and high connection thresholds and concurrent sp retrievals", - args: []string{"daemon", "--libp2p-conns-lowwater", "10", "--libp2p-conns-highwater", "20"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.NotNil(t, lCfg.Host, "host should not be nil") - cmgr, ok := lCfg.Host.ConnManager().(*connmgr.BasicConnMgr) - require.True(t, ok) - cmInfo := cmgr.GetInfo() - require.Equal(t, cmInfo.LowWater, 10) - require.Equal(t, cmInfo.HighWater, 20) - return nil - }, - }, - { - name: "with concurrent sp retrievals", - args: []string{"daemon", "--concurrent-sp-retrievals", "10"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, uint(10), lCfg.ConcurrentSPRetrievals) - return nil - }, - }, - { - name: "with temp directory", - args: []string{"daemon", "--tempdir", "/mytmpdir"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, "/mytmpdir", hCfg.TempDir) - return nil - }, - }, - { - name: "with provider timeout", - args: []string{"daemon", "--provider-timeout", "30s"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, 30*time.Second, lCfg.ProviderTimeout) - return nil - }, - }, - { - name: "with global timeout", - args: []string{"daemon", "--global-timeout", "30s"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, 30*time.Second, lCfg.GlobalTimeout) - return nil - }, - }, - { - name: "with protocols", - args: []string{"daemon", "--protocols", "bitswap,graphsync"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportGraphsyncFilecoinv1}, lCfg.Protocols) - return nil - }, - }, - { - name: "with exclude providers", - args: []string{"daemon", "--exclude-providers", "12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4,12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - p1, err := peer.Decode("12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4") - require.NoError(t, err) - p2, err := peer.Decode("12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys") - require.NoError(t, err) - - require.Equal(t, true, lCfg.ProviderBlockList[p1]) - require.Equal(t, true, lCfg.ProviderBlockList[p2]) - return nil - }, - }, - { - name: "with bitswap concurrency", - args: []string{"daemon", "--bitswap-concurrency", "10"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, 10, lCfg.BitswapConcurrency) - return nil - }, - }, - { - name: "with address", - args: []string{"daemon", "--address", "0.0.0.0"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, "0.0.0.0", hCfg.Address) - return nil - }, - }, - { - name: "with port", - args: []string{"daemon", "--port", "1234"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, uint(1234), hCfg.Port) - return nil - }, - }, - { - name: "with max blocks", - args: []string{"daemon", "--maxblocks", "10"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, uint64(10), hCfg.MaxBlocksPerRequest) - return nil - }, - }, - { - name: "with ipni endpoint", - args: []string{"daemon", "--ipni-endpoint", "https://cid.contact"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.IsType(t, &indexerlookup.IndexerCandidateSource{}, lCfg.Source, "finder should be an IndexerCandidateSource when providing an ipni endpoint") - return nil - }, - }, - { - name: "with bad ipni endpoint", - args: []string{"daemon", "--ipni-endpoint", "not-a-url"}, - shouldError: true, - }, - { - name: "with event recorder url", - args: []string{"daemon", "--event-recorder-url", "https://myeventrecorder.com/v1/retrieval-events"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, "https://myeventrecorder.com/v1/retrieval-events", erCfg.EndpointURL) - return nil - }, - }, - { - name: "with event recorder auth", - args: []string{"daemon", "--event-recorder-auth", "secret"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, "secret", erCfg.EndpointAuthorization) - return nil - }, - }, - { - name: "with event recorder instance ID", - args: []string{"daemon", "--event-recorder-instance-id", "myinstanceid"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, "myinstanceid", erCfg.InstanceID) - return nil - }, - }, - { - name: "with access token", - args: []string{"daemon", "--access-token", "super-secret"}, - assert: func(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - require.Equal(t, "super-secret", hCfg.AccessToken) - return nil - }, - }, - } - - for _, test := range tests { - daemonRun = test.assert - if test.shouldError { - daemonRun = noopDaemonRun - } - - app := &cli.App{ - Name: "cli-test", - Flags: daemonFlags, - Commands: []*cli.Command{daemonCmd}, - } - - t.Run(test.name, func(t *testing.T) { - err := app.Run(append([]string{"cli-test"}, test.args...)) - if err != nil && !test.shouldError { - t.Fatal(err) - } - - if err == nil && test.shouldError { - t.Fatal("expected error") - } - }) - } -} - -func noopDaemonRun(ctx context.Context, lCfg *l.LassieConfig, hCfg h.HttpServerConfig, erCfg *a.EventRecorderConfig) error { - return nil -} diff --git a/cmd/lassie/fetch.go b/cmd/lassie/fetch.go index 7e935f95..d69f2f8b 100644 --- a/cmd/lassie/fetch.go +++ b/cmd/lassie/fetch.go @@ -5,11 +5,15 @@ import ( "fmt" "io" "net/url" + "os" + "runtime/pprof" "strings" + "sync/atomic" "github.com/dustin/go-humanize" "github.com/filecoin-project/lassie/pkg/aggregateeventrecorder" "github.com/filecoin-project/lassie/pkg/events" + "github.com/filecoin-project/lassie/pkg/extractor" "github.com/filecoin-project/lassie/pkg/lassie" "github.com/filecoin-project/lassie/pkg/storage" "github.com/filecoin-project/lassie/pkg/types" @@ -17,7 +21,6 @@ import ( "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage/deferred" "github.com/ipld/go-ipld-prime/datamodel" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" trustlessutils "github.com/ipld/go-trustless-utils" trustlesshttp "github.com/ipld/go-trustless-utils/http" "github.com/urfave/cli/v2" @@ -36,7 +39,12 @@ var fetchFlags = []cli.Flag{ &cli.BoolFlag{ Name: "progress", Aliases: []string{"p"}, - Usage: "print progress output", + Usage: "print verbose progress including provider events", + }, + &cli.BoolFlag{ + Name: "quiet", + Aliases: []string{"q"}, + Usage: "suppress progress output", }, &cli.StringFlag{ Name: "dag-scope", @@ -72,24 +80,35 @@ var fetchFlags = []cli.Flag{ }, }, &cli.BoolFlag{ - Name: "duplicates", - Usage: "allow duplicate blocks to be written to the output CAR, which " + - "may be useful for streaming.", - Aliases: []string{"dups"}, + Name: "stream", + Usage: "stream blocks directly to output; disable to use temp files for deduplication", + Value: true, + Aliases: []string{"s"}, }, - FlagIPNIEndpoint, + FlagDelegatedRoutingEndpoint, FlagEventRecorderAuth, FlagEventRecorderInstanceId, FlagEventRecorderUrl, FlagVerbose, FlagVeryVerbose, - FlagProtocols, FlagAllowProviders, FlagExcludeProviders, FlagTempDir, - FlagBitswapConcurrency, FlagGlobalTimeout, - FlagProviderTimeout, + FlagSkipBlockVerification, + &cli.StringFlag{ + Name: "cpuprofile", + Usage: "write cpu profile to file", + }, + &cli.BoolFlag{ + Name: "extract", + Usage: "extract UnixFS content to files instead of CAR output", + }, + &cli.StringFlag{ + Name: "extract-to", + Usage: "directory to extract files to (default: current directory)", + Value: ".", + }, } var fetchCmd = &cli.Command{ @@ -109,10 +128,22 @@ func fetchAction(cctx *cli.Context) error { return nil } + if cpuprofile := cctx.String("cpuprofile"); cpuprofile != "" { + f, err := os.Create(cpuprofile) + if err != nil { + return fmt.Errorf("could not create CPU profile: %w", err) + } + defer f.Close() + if err := pprof.StartCPUProfile(f); err != nil { + return fmt.Errorf("could not start CPU profile: %w", err) + } + defer pprof.StopCPUProfile() + } + msgWriter := cctx.App.ErrWriter dataWriter := cctx.App.Writer - root, path, scope, byteRange, duplicates, err := parseCidPath(cctx.Args().Get(0)) + root, path, scope, byteRange, stream, err := parseCidPath(cctx.Args().Get(0)) if err != nil { return err } @@ -133,12 +164,13 @@ func fetchAction(cctx *cli.Context) error { } } - if cctx.IsSet("duplicates") { - duplicates = cctx.Bool("duplicates") + if cctx.IsSet("stream") { + stream = cctx.Bool("stream") } tempDir := cctx.String("tempdir") progress := cctx.Bool("progress") + quiet := cctx.Bool("quiet") output := cctx.String("output") outfile := fmt.Sprintf("%s.car", root.String()) @@ -146,7 +178,15 @@ func fetchAction(cctx *cli.Context) error { outfile = output } - lassieCfg, err := buildLassieConfigFromCLIContext(cctx, nil, nil) + extractMode := cctx.Bool("extract") + extractTo := cctx.String("extract-to") + + // validate flags + if extractMode && cctx.IsSet("output") { + return fmt.Errorf("--extract and --output are mutually exclusive") + } + + lassieCfg, err := buildLassieConfigFromCLIContext(cctx, nil) if err != nil { return err } @@ -156,21 +196,38 @@ func fetchAction(cctx *cli.Context) error { instanceID := cctx.String("event-recorder-instance-id") eventRecorderCfg := getEventRecorderConfig(eventRecorderURL, authToken, instanceID) - err = fetchRun( - cctx.Context, - lassieCfg, - eventRecorderCfg, - msgWriter, - dataWriter, - root, - path, - scope, - byteRange, - duplicates, - tempDir, - progress, - outfile, - ) + if extractMode { + err = extractRun( + cctx.Context, + lassieCfg, + eventRecorderCfg, + msgWriter, + root, + path, + scope, + byteRange, + progress, + quiet, + extractTo, + ) + } else { + err = fetchRun( + cctx.Context, + lassieCfg, + eventRecorderCfg, + msgWriter, + dataWriter, + root, + path, + scope, + byteRange, + stream, + tempDir, + progress, + quiet, + outfile, + ) + } if err != nil { return cli.Exit(err, 1) } @@ -183,24 +240,25 @@ func parseCidPath(spec string) ( path datamodel.Path, scope trustlessutils.DagScope, byteRange *trustlessutils.ByteRange, - duplicates bool, + stream bool, err error, ) { scope = trustlessutils.DagScopeAll // default + stream = true // default to streaming mode if !strings.HasPrefix(spec, "/ipfs/") { cstr := strings.Split(spec, "/")[0] path = datamodel.ParsePath(strings.TrimPrefix(spec, cstr)) if root, err = cid.Parse(cstr); err != nil { - return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, false, err + return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, true, err } - return root, path, scope, byteRange, duplicates, err + return root, path, scope, byteRange, stream, err } else { specParts := strings.Split(spec, "?") spec = specParts[0] if root, path, err = trustlesshttp.ParseUrlPath(spec); err != nil { - return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, false, err + return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, true, err } switch len(specParts) { @@ -208,25 +266,27 @@ func parseCidPath(spec string) ( case 2: query, err := url.ParseQuery(specParts[1]) if err != nil { - return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, false, err + return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, true, err } scope, err = trustlessutils.ParseDagScope(query.Get("dag-scope")) if err != nil { - return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, false, err + return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, true, err } if query.Get("entity-bytes") != "" { br, err := trustlessutils.ParseByteRange(query.Get("entity-bytes")) if err != nil { - return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, false, err + return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, true, err } byteRange = &br } - duplicates = query.Get("dups") == "y" + if query.Has("dups") { + stream = query.Get("dups") == "y" + } default: - return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, false, fmt.Errorf("invalid query: %s", spec) + return cid.Undef, datamodel.Path{}, trustlessutils.DagScopeAll, nil, true, fmt.Errorf("invalid query: %s", spec) } - return root, path, scope, byteRange, duplicates, nil + return root, path, scope, byteRange, stream, nil } } @@ -243,22 +303,18 @@ func (pp *progressPrinter) subscriber(event types.RetrievalEvent) { fmt.Fprintf(pp.writer, "\rRetrieving from [%s] (%s)...\n", events.Identifier(ret), ret.Code()) case events.ConnectedToProviderEvent: fmt.Fprintf(pp.writer, "\rRetrieving from [%s] (%s)...\n", events.Identifier(ret), ret.Code()) - case events.GraphsyncProposedEvent: - fmt.Fprintf(pp.writer, "\rRetrieving from [%s] (%s)...\n", events.Identifier(ret), ret.Code()) - case events.GraphsyncAcceptedEvent: - fmt.Fprintf(pp.writer, "\rRetrieving from [%s] (%s)...\n", events.Identifier(ret), ret.Code()) case events.FirstByteEvent: fmt.Fprintf(pp.writer, "\rRetrieving from [%s] (%s)...\n", events.Identifier(ret), ret.Code()) case events.CandidatesFoundEvent: pp.candidatesFound = len(ret.Candidates()) case events.CandidatesFilteredEvent: if len(fetchProviders) == 0 { - fmt.Fprintf(pp.writer, "Found %d storage provider candidate(s) in the indexer:\n", pp.candidatesFound) + fmt.Fprintf(pp.writer, "Found %d provider candidate(s) in the indexer:\n", pp.candidatesFound) } else { - fmt.Fprintf(pp.writer, "Using the specified storage provider(s):\n") + fmt.Fprintf(pp.writer, "Using the specified provider(s):\n") } for _, candidate := range ret.Candidates() { - fmt.Fprintf(pp.writer, "\r\t%s, Protocols: %v\n", candidate.MinerPeer.ID, candidate.Metadata.Protocols()) + fmt.Fprintf(pp.writer, "\r\t%s\n", candidate.Endpoint()) } case events.FailedEvent: fmt.Fprintf(pp.writer, "\rRetrieval failure from indexer: %s\n", ret.ErrorMessage()) @@ -266,6 +322,10 @@ func (pp *progressPrinter) subscriber(event types.RetrievalEvent) { fmt.Fprintf(pp.writer, "\rRetrieval failure for [%s]: %s\n", events.Identifier(ret), ret.ErrorMessage()) case events.SucceededEvent: // noop, handled at return from Retrieve() + case events.ExtractionStartedEvent: + fmt.Fprintf(pp.writer, "Streaming from [%s]...\n", ret.Endpoint()) + case events.ExtractionSucceededEvent: + // noop, handled at return from Extract() } } @@ -287,9 +347,10 @@ type fetchRunFunc func( path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, - duplicates bool, + stream bool, tempDir string, progress bool, + quiet bool, outfile string, ) error @@ -308,19 +369,20 @@ func defaultFetchRun( path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, - duplicates bool, + stream bool, tempDir string, progress bool, + quiet bool, outfile string, ) error { - lassie, err := lassie.NewLassieWithConfig(ctx, lassieCfg) + s, err := lassie.NewLassieWithConfig(ctx, lassieCfg) if err != nil { return err } // create and subscribe an event recorder API if an endpoint URL is set if eventRecorderCfg.EndpointURL != "" { - setupLassieEventRecorder(ctx, eventRecorderCfg, lassie) + setupLassieEventRecorder(ctx, eventRecorderCfg, s) } printPath := path.String() @@ -328,55 +390,61 @@ func defaultFetchRun( printPath = "/" + printPath } if len(fetchProviders) == 0 { - fmt.Fprintf(msgWriter, "Fetching %s", rootCid.String()+printPath) + fmt.Fprintf(msgWriter, "Fetching %s\n", rootCid.String()+printPath) } else { - fmt.Fprintf(msgWriter, "Fetching %s from specified provider(s)", rootCid.String()+printPath) + fmt.Fprintf(msgWriter, "Fetching %s from specified provider(s)\n", rootCid.String()+printPath) } if progress { - fmt.Fprintln(msgWriter) pp := &progressPrinter{writer: msgWriter} - lassie.RegisterSubscriber(pp.subscriber) + s.RegisterSubscriber(pp.subscriber) } - var carWriter storage.DeferredWriter carOpts := []car.Option{ car.WriteAsCarV1(true), car.StoreIdentityCIDs(false), car.UseWholeCIDs(false), } - tempStore := storage.NewDeferredStorageCar(tempDir, rootCid) + var carStore types.ReadableWritableStorage + var carWriter storage.DeferredWriter - if outfile == stdoutFileString { - // we need the onlyWriter because stdout is presented as an os.File, and - // therefore pretend to support seeks, so feature-checking in go-car - // will make bad assumptions about capabilities unless we hide it - w := &onlyWriter{dataWriter} - if duplicates { - carWriter = storage.NewDuplicateAdderCarForStream(ctx, w, rootCid, path.String(), dagScope, entityBytes, tempStore) + if stream { + var deferredWriter *deferred.DeferredCarWriter + streamOpts := []car.Option{ + car.WriteAsCarV1(true), + car.AllowDuplicatePuts(true), + car.StoreIdentityCIDs(false), + car.UseWholeCIDs(true), + } + if outfile == stdoutFileString { + deferredWriter = deferred.NewDeferredCarWriterForStream(&onlyWriter{dataWriter}, []cid.Cid{rootCid}, streamOpts...) } else { - carWriter = deferred.NewDeferredCarWriterForStream(w, []cid.Cid{rootCid}, carOpts...) + deferredWriter = deferred.NewDeferredCarWriterForPath(outfile, []cid.Cid{rootCid}, streamOpts...) } + carWriter = deferredWriter + carStore = storage.NewStreamingStore(deferredWriter.BlockWriteOpener()) } else { - if duplicates { - carWriter = storage.NewDuplicateAdderCarForPath(ctx, outfile, rootCid, path.String(), dagScope, entityBytes, tempStore) + tempStore := storage.NewDeferredStorageCar(tempDir, rootCid) + if outfile == stdoutFileString { + carWriter = deferred.NewDeferredCarWriterForStream(&onlyWriter{dataWriter}, []cid.Cid{rootCid}, carOpts...) } else { carWriter = deferred.NewDeferredCarWriterForPath(outfile, []cid.Cid{rootCid}, carOpts...) } + carStore = storage.NewCachingTempStore(carWriter.BlockWriteOpener(), tempStore) } defer carWriter.Close() - - carStore := storage.NewCachingTempStore(carWriter.BlockWriteOpener(), tempStore) - defer carStore.Close() + defer func() { + if closer, ok := carStore.(interface{ Close() error }); ok { + closer.Close() + } + }() var blockCount int var byteLength uint64 carWriter.OnPut(func(putBytes int) { blockCount++ byteLength += uint64(putBytes) - if !progress { - fmt.Fprint(msgWriter, ".") - } else { + if !quiet && !progress { fmt.Fprintf(msgWriter, "\rReceived %d blocks / %s...", blockCount, humanize.IBytes(byteLength)) } }, false) @@ -385,30 +453,27 @@ func defaultFetchRun( if err != nil { return err } - // setup preload storage for bitswap, the temporary CAR store can set up a - // separate preload space in its storage - request.PreloadLinkSystem = cidlink.DefaultLinkSystem() - preloadStore := carStore.PreloadStore() - request.PreloadLinkSystem.SetReadStorage(preloadStore) - request.PreloadLinkSystem.SetWriteStorage(preloadStore) - request.PreloadLinkSystem.TrustedStorage = true - request.Duplicates = duplicates + request.Duplicates = stream - stats, err := lassie.Fetch(ctx, request) + stats, err := s.Fetch(ctx, request) if err != nil { fmt.Fprintln(msgWriter) return err } - spid := stats.StorageProviderId.String() - if spid == "" { - spid = types.BitswapIndentifier + providers := stats.Providers + if len(providers) == 0 && stats.StorageProviderId.String() != "" { + providers = []string{stats.StorageProviderId.String()} + } + providerStr := "Unknown" + if len(providers) > 0 { + providerStr = strings.Join(providers, ", ") } fmt.Fprintf(msgWriter, "\nFetched [%s] from [%s]:\n"+ "\tDuration: %s\n"+ "\t Blocks: %d\n"+ "\t Bytes: %s\n", rootCid, - spid, + providerStr, stats.Duration, blockCount, humanize.IBytes(stats.Size), @@ -416,3 +481,101 @@ func defaultFetchRun( return nil } + +// extractRun handles extraction mode - extracting UnixFS content to files +func extractRun( + ctx context.Context, + lassieCfg *lassie.LassieConfig, + eventRecorderCfg *aggregateeventrecorder.EventRecorderConfig, + msgWriter io.Writer, + rootCid cid.Cid, + path datamodel.Path, + dagScope trustlessutils.DagScope, + entityBytes *trustlessutils.ByteRange, + progress bool, + quiet bool, + extractTo string, +) error { + // path and scope/byteRange not yet supported in streaming extraction + if path.Len() > 0 { + return fmt.Errorf("path traversal not yet supported in streaming extraction") + } + if dagScope != trustlessutils.DagScopeAll { + return fmt.Errorf("dag-scope other than 'all' not yet supported in streaming extraction") + } + if entityBytes != nil { + return fmt.Errorf("entity-bytes not yet supported in streaming extraction") + } + + s, err := lassie.NewLassieWithConfig(ctx, lassieCfg) + if err != nil { + return err + } + + if eventRecorderCfg.EndpointURL != "" { + setupLassieEventRecorder(ctx, eventRecorderCfg, s) + } + + // create extractor + ext, err := extractor.New(extractTo) + if err != nil { + return fmt.Errorf("failed to create extractor: %w", err) + } + defer ext.Close() + + // set root path context + ext.SetRootPath(rootCid, rootCid.String()) + + printPath := path.String() + if printPath != "" { + printPath = "/" + printPath + } + if len(fetchProviders) == 0 { + fmt.Fprintf(msgWriter, "Extracting %s to %s\n", rootCid.String()+printPath, extractTo) + } else { + fmt.Fprintf(msgWriter, "Extracting %s to %s from specified provider(s)\n", rootCid.String()+printPath, extractTo) + } + if progress { + pp := &progressPrinter{writer: msgWriter} + s.RegisterSubscriber(pp.subscriber) + } + + var blockCount int64 + var byteLength uint64 + onBlock := func(putBytes int) { + bc := atomic.AddInt64(&blockCount, 1) + bl := atomic.AddUint64(&byteLength, uint64(putBytes)) + if !quiet && !progress { + fmt.Fprintf(msgWriter, "\rReceived %d blocks / %s...", bc, humanize.IBytes(bl)) + } + } + + stats, err := s.Extract(ctx, rootCid, ext, nil, onBlock) + if err != nil { + fmt.Fprintln(msgWriter) + return err + } + + providers := stats.Providers + if len(providers) == 0 && stats.StorageProviderId.String() != "" { + providers = []string{stats.StorageProviderId.String()} + } + providerStr := "Unknown" + if len(providers) > 0 { + providerStr = strings.Join(providers, ", ") + } + fmt.Fprintf(msgWriter, "\nExtracted [%s] from [%s]:\n"+ + "\tDuration: %s\n"+ + "\t Blocks: %d\n"+ + "\t Bytes: %s\n"+ + "\t To: %s\n", + rootCid, + providerStr, + stats.Duration, + atomic.LoadInt64(&blockCount), + humanize.IBytes(atomic.LoadUint64(&byteLength)), + extractTo, + ) + + return nil +} diff --git a/cmd/lassie/fetch_test.go b/cmd/lassie/fetch_test.go deleted file mode 100644 index b5d2616e..00000000 --- a/cmd/lassie/fetch_test.go +++ /dev/null @@ -1,473 +0,0 @@ -package main - -import ( - "context" - "io" - "testing" - "time" - - a "github.com/filecoin-project/lassie/pkg/aggregateeventrecorder" - "github.com/filecoin-project/lassie/pkg/indexerlookup" - l "github.com/filecoin-project/lassie/pkg/lassie" - "github.com/filecoin-project/lassie/pkg/retriever" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/datamodel" - trustlessutils "github.com/ipld/go-trustless-utils" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multicodec" - "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" -) - -var emptyPath = datamodel.ParsePath("") - -func TestFetchCommandFlags(t *testing.T) { - tests := []struct { - name string - args []string - shouldError bool - assertRun fetchRunFunc - }{ - { - name: "with default args", - args: []string{"fetch", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"}, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - // fetch specific params - require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String()) - require.Equal(t, emptyPath, path) - require.Equal(t, trustlessutils.DagScopeAll, dagScope) - require.Nil(t, entityBytes) - require.False(t, duplicates) - require.False(t, progress) - require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4.car", outfile) - - // lassie config - require.Equal(t, nil, lCfg.Source) - require.NotNil(t, lCfg.Host, "host should not be nil") - require.Equal(t, 20*time.Second, lCfg.ProviderTimeout) - require.Equal(t, uint(0), lCfg.ConcurrentSPRetrievals) - require.Equal(t, 0*time.Second, lCfg.GlobalTimeout) - require.Equal(t, 0, len(lCfg.Libp2pOptions)) - require.Equal(t, 0, len(lCfg.Protocols)) - require.Equal(t, 0, len(lCfg.ProviderBlockList)) - require.Equal(t, 0, len(lCfg.ProviderAllowList)) - // there's only one --bitswap-concurrency for `fetch` and it sets both to be the same - require.Equal(t, 32, lCfg.BitswapConcurrency) - require.Equal(t, 32, lCfg.BitswapConcurrencyPerRetrieval) - - // event recorder config - require.Equal(t, "", erCfg.EndpointURL) - require.Equal(t, "", erCfg.EndpointAuthorization) - require.Equal(t, "", erCfg.InstanceID) - return nil - }, - }, - { - name: "with bad root cid", - args: []string{ - "fetch", - "not-a-cid", - }, - shouldError: true, - }, - { - name: "with root cid path", - args: []string{ - "fetch", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, datamodel.ParsePath("birb.mp4"), path) - return nil - }, - }, - { - name: "with dag scope entity", - args: []string{ - "fetch", - "--dag-scope", - "entity", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, trustlessutils.DagScopeEntity, dagScope) - return nil - }, - }, - { - name: "with dag scope block", - args: []string{ - "fetch", - "--dag-scope", - "block", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, trustlessutils.DagScopeBlock, dagScope) - return nil - }, - }, - { - name: "with entity-bytes 0:*", - args: []string{ - "fetch", - "--entity-bytes", - "0:*", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Nil(t, entityBytes) // default is ignored - return nil - }, - }, - { - name: "with entity-bytes 0:10", - args: []string{ - "fetch", - "--entity-bytes", - "0:10", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - var to int64 = 10 - require.Equal(t, &trustlessutils.ByteRange{From: 0, To: &to}, entityBytes) - return nil - }, - }, - { - name: "with entity-bytes 1000:20000", - args: []string{ - "fetch", - "--entity-bytes", - "1000:20000", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - var to int64 = 20000 - require.Equal(t, &trustlessutils.ByteRange{From: 1000, To: &to}, entityBytes) - return nil - }, - }, - { - name: "with duplicates", - args: []string{ - "fetch", - "--duplicates", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.True(t, duplicates) - return nil - }, - }, - { - name: "with progress", - args: []string{ - "fetch", - "--progress", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.True(t, progress) - return nil - }, - }, - { - name: "with output", - args: []string{ - "fetch", - "--output", - "myfile", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, "myfile", outfile) - return nil - }, - }, - { - name: "with providers", - args: []string{ - "fetch", - "--providers", - "/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.IsType(t, &retriever.DirectCandidateSource{}, lCfg.Source, "finder should be a DirectCandidateSource when providers are specified") - return nil - }, - }, - { - name: "with ipni endpoint", - args: []string{ - "fetch", - "--ipni-endpoint", - "https://cid.contact", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.IsType(t, &indexerlookup.IndexerCandidateSource{}, lCfg.Source, "finder should be an IndexerCandidateSource when providing an ipni endpoint") - return nil - }, - }, - { - name: "with bad ipni endpoint", - args: []string{ - "fetch", - "--ipni-endpoint", - "not-a-url", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - shouldError: true, - }, - { - name: "with temp directory", - args: []string{ - "fetch", - "--tempdir", - "/mytmpdir", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, "/mytmpdir", tempDir) - return nil - }, - }, - { - name: "with provider timeout", - args: []string{ - "fetch", - "--provider-timeout", - "30s", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, 30*time.Second, lCfg.ProviderTimeout) - return nil - }, - }, - { - name: "with global timeout", - args: []string{ - "fetch", - "--global-timeout", - "30s", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, 30*time.Second, lCfg.GlobalTimeout) - return nil - }, - }, - { - name: "with protocols", - args: []string{ - "fetch", - "--protocols", - "bitswap,graphsync", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportGraphsyncFilecoinv1}, lCfg.Protocols) - return nil - }, - }, - { - name: "with exclude providers", - args: []string{ - "fetch", - "--exclude-providers", - "12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4,12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - p1, err := peer.Decode("12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4") - require.NoError(t, err) - p2, err := peer.Decode("12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys") - require.NoError(t, err) - - require.True(t, lCfg.ProviderBlockList[p1]) - require.True(t, lCfg.ProviderBlockList[p2]) - return nil - }, - }, - { - name: "with bitswap concurrency", - args: []string{ - "fetch", - "--bitswap-concurrency", - "10", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, 10, lCfg.BitswapConcurrency) - return nil - }, - }, - { - name: "with event recorder url", - args: []string{ - "fetch", - "--event-recorder-url", - "https://myeventrecorder.com/v1/retrieval-events", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, "https://myeventrecorder.com/v1/retrieval-events", erCfg.EndpointURL) - return nil - }, - }, - { - name: "with event recorder auth", - args: []string{ - "fetch", - "--event-recorder-auth", - "secret", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, "secret", erCfg.EndpointAuthorization) - return nil - }, - }, - { - name: "with event recorder instance ID", - args: []string{ - "fetch", - "--event-recorder-instance-id", - "myinstanceid", - "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - require.Equal(t, "myinstanceid", erCfg.InstanceID) - return nil - }, - }, - { - name: "with trustless url", - args: []string{ - "fetch", - "/ipfs/bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - // fetch specific params - require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String()) - require.Equal(t, emptyPath, path) - require.Equal(t, trustlessutils.DagScopeAll, dagScope) - require.Nil(t, entityBytes) - return nil - }, - }, - { - name: "with trustless url+path", - args: []string{ - "fetch", - "/ipfs/bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4/nope", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - // fetch specific params - require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String()) - require.Equal(t, datamodel.ParsePath("birb.mp4/nope"), path) - require.Equal(t, trustlessutils.DagScopeAll, dagScope) - require.Nil(t, entityBytes) - return nil - }, - }, - { - name: "with trustless url+path+scope", - args: []string{ - "fetch", - "/ipfs/bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4/nope?dag-scope=entity", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - // fetch specific params - require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String()) - require.Equal(t, datamodel.ParsePath("birb.mp4/nope"), path) - require.Equal(t, trustlessutils.DagScopeEntity, dagScope) - require.Nil(t, entityBytes) - return nil - }, - }, - { - name: "with trustless url+path+scope+entity-bytes", - args: []string{ - "fetch", - "/ipfs/bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4/nope?dag-scope=entity&entity-bytes=1000:20000", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - // fetch specific params - require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String()) - require.Equal(t, datamodel.ParsePath("birb.mp4/nope"), path) - require.Equal(t, trustlessutils.DagScopeEntity, dagScope) - var to int64 = 20000 - require.Equal(t, &trustlessutils.ByteRange{From: 1000, To: &to}, entityBytes) - return nil - }, - }, - { - name: "with trustless url+path+scope+entity-bytes w/ overrides", - args: []string{ - "fetch", - "--dag-scope", "block", - "--entity-bytes", "0:*", - "/ipfs/bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4/nope?dag-scope=entity&entity-bytes=1000:20000", - }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path datamodel.Path, dagScope trustlessutils.DagScope, entityBytes *trustlessutils.ByteRange, duplicates bool, tempDir string, progress bool, outfile string) error { - // fetch specific params - require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String()) - require.Equal(t, datamodel.ParsePath("birb.mp4/nope"), path) - require.Equal(t, trustlessutils.DagScopeBlock, dagScope) - require.Nil(t, entityBytes) - return nil - }, - }, - } - - fetchRunOrig := fetchRun - defer func() { - fetchRun = fetchRunOrig - }() - for _, test := range tests { - // fetchRun is a global var that we can override for testing purposes - fetchRun = test.assertRun - if test.shouldError { - fetchRun = noopRun - } - - app := &cli.App{ - Name: "cli-test", - Flags: fetchFlags, - Commands: []*cli.Command{fetchCmd}, - } - - t.Run(test.name, func(t *testing.T) { - err := app.Run(append([]string{"cli-test"}, test.args...)) - if err != nil && !test.shouldError { - t.Fatal(err) - } - - if err == nil && test.shouldError { - t.Fatal("expected error") - } - }) - } -} - -func noopRun( - ctx context.Context, - lCfg *l.LassieConfig, - erCfg *a.EventRecorderConfig, - msgWriter io.Writer, - dataWriter io.Writer, - rootCid cid.Cid, - path datamodel.Path, - dagScope trustlessutils.DagScope, - entityBytes *trustlessutils.ByteRange, - duplicates bool, - tempDir string, - progress bool, - outfile string, -) error { - return nil -} diff --git a/cmd/lassie/flags.go b/cmd/lassie/flags.go index cca17ab5..ecaa8e29 100644 --- a/cmd/lassie/flags.go +++ b/cmd/lassie/flags.go @@ -3,32 +3,24 @@ package main import ( "os" "strings" - "time" - "github.com/filecoin-project/lassie/pkg/heyfil" - "github.com/filecoin-project/lassie/pkg/lassie" "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multicodec" "github.com/urfave/cli/v2" ) var ( defaultTempDirectory string = os.TempDir() // use the system default temp dir verboseLoggingSubsystems []string = []string{ // verbose logging is enabled for these subsystems when using the verbose or very-verbose flags - "lassie", + "lassie/main", "lassie/retriever", "lassie/httpserver", "lassie/indexerlookup", - "lassie/bitswap", + "lassie/aggregateeventrecorder", } ) -const ( - defaultProviderTimeout time.Duration = 20 * time.Second // 20 seconds -) - // FlagVerbose enables verbose mode, which shows info information about // operations invoked in the CLI. var FlagVerbose = &cli.BoolFlag{ @@ -96,7 +88,7 @@ var providerBlockList map[peer.ID]bool var FlagExcludeProviders = &cli.StringFlag{ Name: "exclude-providers", DefaultText: "All providers allowed", - Usage: "Provider peer IDs, separated by a comma. Example: 12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", + Usage: "Provider peer IDs to exclude, separated by a comma. Note: peer IDs are opaque identifiers from the delegated router. Example: 12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", EnvVars: []string{"LASSIE_EXCLUDE_PROVIDERS"}, Action: func(cctx *cli.Context, v string) error { // Do nothing if given an empty string @@ -123,45 +115,16 @@ var FlagAllowProviders = &cli.StringFlag{ Name: "providers", Aliases: []string{"provider"}, DefaultText: "Providers will be discovered automatically", - Usage: "Comma-separated addresses of providers, to use instead of " + - "automatic discovery. Accepts full multiaddrs including peer ID, " + - "multiaddrs without peer ID and url-style addresses for HTTP and " + - "Filecoin SP f0 actor addresses. Lassie will attempt to connect to the " + - "peer(s). Example: " + - "/ip4/1.2.3.4/tcp/1234/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4,http://ipfs.io,f01234", + Usage: "Comma-separated addresses of HTTP gateways to use instead of " + + "automatic discovery. Accepts HTTP URLs and multiaddrs with /http or /https. " + + "Example: https://ipfs.io,/dns/gateway.example.com/tcp/443/https", EnvVars: []string{"LASSIE_ALLOW_PROVIDERS"}, Action: func(cctx *cli.Context, v string) error { - // Do nothing if given an empty string if v == "" { return nil } - - // in case we have been given filecoin actor addresses we can look them up - // with heyfil and translate to full multiaddrs, otherwise this is a - // pass-through - trans, err := heyfil.Heyfil{TranslateFaddr: true}.TranslateAll(strings.Split(v, ",")) - if err != nil { - return err - } - fetchProviders, err = types.ParseProviderStrings(strings.Join(trans, ",")) - return err - }, -} - -var protocols []multicodec.Code -var FlagProtocols = &cli.StringFlag{ - Name: "protocols", - DefaultText: "bitswap,graphsync,http", - Usage: "List of retrieval protocols to use, separated by a comma", - EnvVars: []string{"LASSIE_SUPPORTED_PROTOCOLS"}, - Action: func(cctx *cli.Context, v string) error { - // Do nothing if given an empty string - if v == "" { - return nil - } - var err error - protocols, err = types.ParseProtocolsString(v) + fetchProviders, err = types.ParseProviderStrings(v) return err }, } @@ -175,20 +138,6 @@ var FlagTempDir = &cli.StringFlag{ EnvVars: []string{"LASSIE_TEMP_DIRECTORY"}, } -var FlagBitswapConcurrency = &cli.IntFlag{ - Name: "bitswap-concurrency", - Usage: "maximum number of concurrent bitswap requests", - Value: lassie.DefaultBitswapConcurrency, - EnvVars: []string{"LASSIE_BITSWAP_CONCURRENCY"}, -} - -var FlagBitswapConcurrencyPerRetrieval = &cli.IntFlag{ - Name: "bitswap-concurrency-per-retrieval", - Usage: "maximum number of concurrent bitswap requests per retrieval", - Value: lassie.DefaultBitswapConcurrencyPerRetrieval, - EnvVars: []string{"LASSIE_BITSWAP_CONCURRENCY_PER_RETRIEVAL"}, -} - var FlagGlobalTimeout = &cli.DurationFlag{ Name: "global-timeout", Aliases: []string{"gt"}, @@ -196,25 +145,25 @@ var FlagGlobalTimeout = &cli.DurationFlag{ EnvVars: []string{"LASSIE_GLOBAL_TIMEOUT"}, } -var FlagProviderTimeout = &cli.DurationFlag{ - Name: "provider-timeout", - Aliases: []string{"pt"}, - Usage: "consider it an error after not receiving a response from a storage provider after this amount of time", - Value: defaultProviderTimeout, - EnvVars: []string{"LASSIE_PROVIDER_TIMEOUT"}, +var FlagDelegatedRoutingEndpoint = &cli.StringFlag{ + Name: "delegated-routing-endpoint", + Aliases: []string{"delegated"}, + DefaultText: "Defaults to https://cid.contact", + Usage: "HTTP endpoint of the delegated routing service used to discover providers.", + EnvVars: []string{"LASSIE_DELEGATED_ROUTING_ENDPOINT"}, } -var FlagIPNIEndpoint = &cli.StringFlag{ - Name: "ipni-endpoint", - Aliases: []string{"ipni"}, - DefaultText: "Defaults to https://cid.contact", - Usage: "HTTP endpoint of the IPNI instance used to discover providers.", +// FlagSkipBlockVerification disables per-block hash verification. +// WARNING: This is dangerous and should only be used for benchmarking. +var FlagSkipBlockVerification = &cli.BoolFlag{ + Name: "skip-block-verification", + Usage: "DANGEROUS: skip per-block hash verification. Malicious gateways can serve arbitrary data!", + EnvVars: []string{"LASSIE_SKIP_BLOCK_VERIFICATION"}, } func ResetGlobalFlags() { // Reset global variables here so that they are not used // in subsequent calls to commands during testing. fetchProviders = make([]types.Provider, 0) - protocols = make([]multicodec.Code, 0) providerBlockList = make(map[peer.ID]bool) } diff --git a/cmd/lassie/main.go b/cmd/lassie/main.go index 2bda11c2..44fea237 100644 --- a/cmd/lassie/main.go +++ b/cmd/lassie/main.go @@ -11,11 +11,9 @@ import ( "github.com/filecoin-project/lassie/pkg/aggregateeventrecorder" "github.com/filecoin-project/lassie/pkg/indexerlookup" "github.com/filecoin-project/lassie/pkg/lassie" - "github.com/filecoin-project/lassie/pkg/net/host" "github.com/filecoin-project/lassie/pkg/retriever" "github.com/google/uuid" "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p/config" "github.com/urfave/cli/v2" ) @@ -45,7 +43,7 @@ func main() { app := &cli.App{ Name: "lassie", - Usage: "Utility for retrieving content from the Filecoin network", + Usage: "Lassie - Utility for retrieving content from the Filecoin network", Suggest: true, Flags: []cli.Flag{ FlagVerbose, @@ -68,62 +66,42 @@ func after(cctx *cli.Context) error { return nil } -func buildLassieConfigFromCLIContext(cctx *cli.Context, lassieOpts []lassie.LassieOption, libp2pOpts []config.Option) (*lassie.LassieConfig, error) { - providerTimeout := cctx.Duration("provider-timeout") +func buildLassieConfigFromCLIContext(cctx *cli.Context, lassieOpts []lassie.LassieOption) (*lassie.LassieConfig, error) { globalTimeout := cctx.Duration("global-timeout") - bitswapConcurrency := cctx.Int("bitswap-concurrency") - bitswapConcurrencyPerRetrieval := cctx.Int("bitswap-concurrency-per-retrieval") - - lassieOpts = append(lassieOpts, lassie.WithProviderTimeout(providerTimeout)) if globalTimeout > 0 { lassieOpts = append(lassieOpts, lassie.WithGlobalTimeout(globalTimeout)) } - if len(protocols) > 0 { - lassieOpts = append(lassieOpts, lassie.WithProtocols(protocols)) - } - - host, err := host.InitHost(cctx.Context, libp2pOpts) - if err != nil { - return nil, err - } - lassieOpts = append(lassieOpts, lassie.WithHost(host)) - if len(fetchProviders) > 0 { - finderOpt := lassie.WithCandidateSource(retriever.NewDirectCandidateSource(fetchProviders, retriever.WithLibp2pCandidateDiscovery(host))) - if cctx.IsSet("ipni-endpoint") { - logger.Warn("Ignoring ipni-endpoint flag since direct provider is specified") + finderOpt := lassie.WithCandidateSource(retriever.NewDirectCandidateSource(fetchProviders)) + if cctx.IsSet("delegated-routing-endpoint") { + logger.Warn("Ignoring delegated-routing-endpoint flag since direct provider is specified") } lassieOpts = append(lassieOpts, finderOpt) - } else if cctx.IsSet("ipni-endpoint") { - endpoint := cctx.String("ipni-endpoint") + } else if cctx.IsSet("delegated-routing-endpoint") { + endpoint := cctx.String("delegated-routing-endpoint") endpointUrl, err := url.ParseRequestURI(endpoint) if err != nil { - logger.Errorw("Failed to parse IPNI endpoint as URL", "err", err) - return nil, fmt.Errorf("cannot parse given IPNI endpoint %s as valid URL: %w", endpoint, err) + logger.Errorw("Failed to parse delegated routing endpoint as URL", "err", err) + return nil, fmt.Errorf("cannot parse given delegated routing endpoint %s as valid URL: %w", endpoint, err) } finder, err := indexerlookup.NewCandidateSource(indexerlookup.WithHttpEndpoint(endpointUrl)) if err != nil { - logger.Errorw("Failed to instantiate IPNI candidate finder", "err", err) + logger.Errorw("Failed to instantiate delegated routing candidate finder", "err", err) return nil, err } lassieOpts = append(lassieOpts, lassie.WithCandidateSource(finder)) - logger.Debug("Using explicit IPNI endpoint to find candidates", "endpoint", endpoint) + logger.Debug("Using explicit delegated routing endpoint to find candidates", "endpoint", endpoint) } if len(providerBlockList) > 0 { lassieOpts = append(lassieOpts, lassie.WithProviderBlockList(providerBlockList)) } - if bitswapConcurrency > 0 { - lassieOpts = append(lassieOpts, lassie.WithBitswapConcurrency(bitswapConcurrency)) - } - - if bitswapConcurrencyPerRetrieval > 0 { - lassieOpts = append(lassieOpts, lassie.WithBitswapConcurrencyPerRetrieval(bitswapConcurrencyPerRetrieval)) - } else if bitswapConcurrency > 0 { - lassieOpts = append(lassieOpts, lassie.WithBitswapConcurrencyPerRetrieval(bitswapConcurrency)) + if cctx.Bool("skip-block-verification") { + logger.Warn("DANGER: block verification disabled - malicious gateways can serve arbitrary data!") + lassieOpts = append(lassieOpts, lassie.WithSkipBlockVerification(true)) } return lassie.NewLassieConfig(lassieOpts...), nil @@ -141,7 +119,7 @@ func getEventRecorderConfig(endpointURL string, authToken string, instanceID str func setupLassieEventRecorder( ctx context.Context, cfg *aggregateeventrecorder.EventRecorderConfig, - lassie *lassie.Lassie, + s *lassie.Lassie, ) { if cfg.EndpointURL != "" { if cfg.InstanceID == "" { @@ -153,7 +131,7 @@ func setupLassieEventRecorder( } eventRecorder := aggregateeventrecorder.NewAggregateEventRecorder(ctx, *cfg) - lassie.RegisterSubscriber(eventRecorder.RetrievalEventSubscriber()) + s.RegisterSubscriber(eventRecorder.RetrievalEventSubscriber()) logger.Infow("Reporting retrieval events to event recorder API", "url", cfg.EndpointURL, "instance_id", cfg.InstanceID) } } diff --git a/docs/CAR.md b/docs/CAR.md index 75f00632..6d382891 100644 --- a/docs/CAR.md +++ b/docs/CAR.md @@ -65,4 +65,3 @@ A "full" (e.g. when using `depthType=full` with the (HTTP)[HTTP_SPEC.md] server) Under certain conditions, a complete DAG may not be returned. * When a Filecoin storage provider chosen to retrieve from does not have the full DAG, only the portion that they do have will be included. - * When a Bitswap session is unable to exhaustively discover all blocks in a DAG, only the portion that was discovered will be included. diff --git a/docs/HTTP_SPEC.md b/docs/HTTP_SPEC.md index fa6714c7..baa8f4e7 100644 --- a/docs/HTTP_SPEC.md +++ b/docs/HTTP_SPEC.md @@ -14,7 +14,7 @@ - [Kyle Huntsman](https://github.com/kylehuntsman) - [Rod Vagg](https://github.com/rvagg) -The Lassie HTTP specification is an HTTP interface for retrieving IPLD data from IPFS and Filecoin peers. It fetches content over the GraphSync, Bitswap, and HTTP protocols and provides the resulting data in CAR format. +The Lassie HTTP specification is an HTTP interface for retrieving IPLD data from IPFS and Filecoin peers. It fetches content over the HTTP protocol and provides the resulting data in CAR format. An implementation of the [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway) specification with a subset of the [Path Gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) specification that focuses on returning verifiable CAR formatted data. @@ -191,15 +191,14 @@ Examples: ### `protocols` (request query parameter) -_OPTIONAL_. `protocols=`. Defaults to all of the specified protocols returned by [IPNI](https://github.com/ipni/specs/blob/main/IPNI.md). +_OPTIONAL_. `protocols=`. Defaults to HTTP. -Used to specify any of the retrieval protocols to use via a comma delimited list. Unrecognized protocols will respond with a 400 status code. +Used to specify the retrieval protocol to use. Unrecognized protocols will respond with a 400 status code. The `protocols` query parameter is a Lassie specific query parameter and is not part of the [Path Gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) specification. Examples: -- `protocols=bitswap` will only attempt retrievals with the bitswap protocol -- `protocols=graphsync,http` will attempt retrievals with both the graphsync and http protocols +- `protocols=http` will attempt retrievals with the HTTP protocol ### `providers` (request query parameter) diff --git a/go.mod b/go.mod index 071b109a..12266dde 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/filecoin-project/lassie go 1.24.6 +toolchain go1.24.9 + require ( github.com/dustin/go-humanize v1.0.1 github.com/filecoin-project/go-clock v0.1.0 @@ -9,12 +11,11 @@ require ( github.com/filecoin-project/go-retrieval-types v1.2.0 github.com/filecoin-project/go-state-types v0.15.0 github.com/google/uuid v1.6.0 - github.com/hannahhoward/go-pubsub v1.0.0 + github.com/ipfs/boxo v0.35.0 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.5.0 - github.com/ipfs/go-datastore v0.9.0 - github.com/ipfs/go-graphsync v0.18.0 github.com/ipfs/go-ipfs-blocksutil v0.0.2 + github.com/ipfs/go-ipld-format v0.6.3 github.com/ipfs/go-log/v2 v2.8.2 github.com/ipfs/go-unixfsnode v1.10.2 github.com/ipld/go-car/v2 v2.15.0 @@ -25,133 +26,68 @@ require ( github.com/libp2p/go-libp2p v0.46.0 github.com/mitchellh/go-server-timing v1.0.1 github.com/multiformats/go-multiaddr v0.16.1 + github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multicodec v0.9.2 github.com/multiformats/go-multihash v0.2.3 github.com/stretchr/testify v1.11.1 github.com/urfave/cli/v2 v2.27.5 - go.opentelemetry.io/otel v1.38.0 - go.opentelemetry.io/otel/trace v1.38.0 go.uber.org/multierr v1.11.0 ) require ( - github.com/benbjohnson/clock v1.3.5 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/bep/debounce v1.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/filecoin-project/go-address v1.2.0 // indirect github.com/filecoin-project/go-amt-ipld/v4 v4.4.0 // indirect github.com/filecoin-project/go-cbor-util v0.0.1 // indirect - github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 // indirect github.com/filecoin-project/go-statemachine v1.0.2 // indirect github.com/filecoin-project/go-statestore v0.2.0 // indirect - github.com/flynn/noise v1.1.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect + github.com/gammazero/chanqueue v1.1.1 // indirect + github.com/gammazero/deque v1.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect - github.com/huin/goupnp v1.3.0 // indirect - github.com/ipfs/boxo v0.35.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect - github.com/ipfs/go-ipfs-pq v0.0.3 // indirect + github.com/ipfs/go-cidutil v0.1.0 // indirect + github.com/ipfs/go-datastore v0.9.0 // indirect + github.com/ipfs/go-dsqueue v0.0.5 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect - github.com/ipfs/go-ipld-format v0.6.3 // indirect github.com/ipfs/go-log v1.0.5 // indirect - github.com/ipfs/go-peertaskqueue v0.8.2 // indirect - github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect - github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/koron/go-ssdp v0.0.6 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect - github.com/libp2p/go-flow-metrics v0.3.0 // indirect - github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-netroute v0.3.0 // indirect - github.com/libp2p/go-reuseport v0.4.0 // indirect - github.com/libp2p/go-yamux/v5 v5.0.1 // indirect - github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.68 // indirect - github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect - github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect - github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multistream v0.6.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect - github.com/pion/datachannel v1.5.10 // indirect - github.com/pion/dtls/v2 v2.2.12 // indirect - github.com/pion/dtls/v3 v3.0.11 // indirect - github.com/pion/ice/v4 v4.0.10 // indirect - github.com/pion/interceptor v0.1.40 // indirect - github.com/pion/logging v0.2.4 // indirect - github.com/pion/mdns/v2 v2.0.7 // indirect - github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.15 // indirect - github.com/pion/rtp v1.8.19 // indirect - github.com/pion/sctp v1.8.39 // indirect - github.com/pion/sdp/v3 v3.0.13 // indirect - github.com/pion/srtp/v3 v3.0.6 // indirect - github.com/pion/stun v0.6.1 // indirect - github.com/pion/stun/v3 v3.0.0 // indirect - github.com/pion/transport/v2 v2.2.10 // indirect - github.com/pion/transport/v3 v3.0.7 // indirect - github.com/pion/transport/v4 v4.0.1 // indirect - github.com/pion/turn/v4 v4.0.2 // indirect - github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.17.0 // indirect - github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.57.1 // indirect - github.com/quic-go/webtransport-go v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.3.1 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect - github.com/wlynxg/anet v0.0.5 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.uber.org/atomic v1.11.0 // indirect - go.uber.org/dig v1.19.0 // indirect - go.uber.org/fx v1.24.0 // indirect - go.uber.org/mock v0.5.2 // indirect go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect golang.org/x/mod v0.29.0 // indirect - golang.org/x/net v0.47.0 // indirect golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect - golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect - golang.org/x/text v0.31.0 // indirect - golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.38.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/protobuf v1.36.9 // indirect diff --git a/go.sum b/go.sum index 9832ada3..61f3c6b4 100644 --- a/go.sum +++ b/go.sum @@ -1,59 +1,30 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= -github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= -github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= +github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -70,22 +41,17 @@ github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/ github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI= github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc8 h1:EWC89lM/tJAjyzaxZ624clq3oyHLoLjISfoyG+WIu9s= github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc8/go.mod h1:mK3/NbSljx3Kr335+IXEe8gcdEPA2eZXJaNhodK9bAI= -github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w= -github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 h1:nYs6OPUF8KbZ3E8o9p9HJnQaE8iugjHR5WYVMcicDJc= github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0/go.mod h1:s0qiHRhFyrgW0SvdQMSJFQxNa4xEIG5XvqCBZUEgcbc= github.com/filecoin-project/go-retrieval-types v1.2.0 h1:fz6DauLVP3GRg7UuW7HZ6sE+GTmaUW70DTXBF1r9cK0= github.com/filecoin-project/go-retrieval-types v1.2.0/go.mod h1:ojW6wSw2GPyoRDBGqw1K6JxUcbfa5NOSIiyQEeh7KK0= github.com/filecoin-project/go-state-types v0.15.0 h1:GaUSCti0tGMzLg7fVpRjtNVGBvirbMFzLfyWbR+qzWE= github.com/filecoin-project/go-state-types v0.15.0/go.mod h1:2okQFn4DVOt5Bs6OFh0lLSzn8p7Vczh8XjgaKLKhKgI= -github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v1.0.2 h1:421SSWBk8GIoCoWYYTE/d+qCWccgmRH0uXotXRDjUbc= github.com/filecoin-project/go-statemachine v1.0.2/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNdofHZoGPjfNaAo5Q= github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= -github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= -github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/gammazero/chanqueue v1.1.1 h1:n9Y+zbBxw2f7uUE9wpgs0rOSkP/I/yhDLiNuhyVjojQ= @@ -93,9 +59,6 @@ github.com/gammazero/chanqueue v1.1.1/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+E github.com/gammazero/deque v1.1.0 h1:OyiyReBbnEG2PP0Bnv1AASLIYvyKqIFN5xfl1t8oGLo= github.com/gammazero/deque v1.1.0/go.mod h1:JVrR+Bj1NMQbPnYclvDlvSX0nVGReLrQZ0aUMuWLctg= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -107,58 +70,28 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 h1:yrv1uUvgXH/tEat+wdvJMRJ4g51GlIydtDpU9pFjaaI= github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c h1:iiD+p+U0M6n/FsO6XIZuOgobnNa48FxtyYFfWwLttUQ= github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= -github.com/hannahhoward/go-pubsub v1.0.0 h1:yONMbY9blu+FFlamGzRZVocoY6WHPJa08h3yX7nOGuA= -github.com/hannahhoward/go-pubsub v1.0.0/go.mod h1:3lHsAt5uM7YFHauT5whoifwfgIgVwEX2fMDxPDrkpU4= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.35.0 h1:3Mku5arSbAZz0dvb4goXRsQuZkFkPrGr5yYdu0YM1pY= @@ -179,20 +112,19 @@ github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= -github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.9.0 h1:WocriPOayqalEsueHv6SdD4nPVl4rYMfYGLD4bqCZ+w= github.com/ipfs/go-datastore v0.9.0/go.mod h1:uT77w/XEGrvJWwHgdrMr8bqCN6ZTW9gzmi+3uK+ouHg= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-dsqueue v0.0.5 h1:TUOk15TlCJ/NKV8Yk2W5wgkEjDa44Nem7a7FGIjsMNU= github.com/ipfs/go-dsqueue v0.0.5/go.mod h1:i/jAlpZjBbQJLioN+XKbFgnd+u9eAhGZs9IrqIzTd9g= -github.com/ipfs/go-graphsync v0.18.0 h1:b+DNJ4lWsCKKaVKYgqgt4rrqshvVcTphN7Rl0JfFiD4= -github.com/ipfs/go-graphsync v0.18.0/go.mod h1:+7SU0L6thSFFTo1pbDcwCqi+gN0L7UQP8eVm897Mg0s= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.2 h1:hoAV68CQOzUa/e1egCME3lbrsyEGO0pY7Bb26T+8/Zc= github.com/ipfs/go-ipfs-blocksutil v0.0.2/go.mod h1:C+qFmwtqDnov1C8ItGpWkN4iHLS1DMzNzy3pCZJ8YdM= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= +github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= @@ -208,7 +140,6 @@ github.com/ipfs/go-ipld-format v0.6.3/go.mod h1:74ilVN12NXVMIV+SrBAyC05UJRk0jVvG github.com/ipfs/go-ipld-legacy v0.2.2 h1:DThbqCPVLpWBcGtU23KDLiY2YRZZnTkXQyfz8aOfBkQ= github.com/ipfs/go-ipld-legacy v0.2.2/go.mod h1:hhkj+b3kG9b2BcUNw8IFYAsfeNo8E3U7eYlWeAOPyDU= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= -github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= @@ -242,15 +173,9 @@ github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+ github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4= github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU= -github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= -github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -264,7 +189,6 @@ github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -279,30 +203,18 @@ github.com/libp2p/go-libp2p v0.46.0 h1:0T2yvIKpZ3DVYCuPOFxPD1layhRU486pj9rSlGWYn github.com/libp2p/go-libp2p v0.46.0/go.mod h1:TbIDnpDjBLa7isdgYpbxozIVPBTmM/7qKOJP4SFySrQ= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= +github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-netroute v0.3.0 h1:nqPCXHmeNmgTJnktosJ/sIef9hvwYCrsLxXmfNks/oc= github.com/libp2p/go-netroute v0.3.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= -github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= -github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= -github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= -github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= -github.com/marcopolo/simnet v0.0.1 h1:rSMslhPz6q9IvJeFWDoMGxMIrlsbXau3NkuIXHGJxfg= -github.com/marcopolo/simnet v0.0.1/go.mod h1:WDaQkgLAjqDUEBAOXz22+1j6wXKfGlC5sD5XWt3ddOs= -github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= -github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= -github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= -github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= -github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= -github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= -github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= -github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -322,7 +234,6 @@ github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYg github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= @@ -336,7 +247,6 @@ github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6o github.com/multiformats/go-multicodec v0.9.2 h1:YrlXCuqxjqm3bXl+vBq5LKz5pz4mvAsugdqy78k0pXQ= github.com/multiformats/go-multicodec v0.9.2/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= -github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= @@ -351,54 +261,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= -github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= -github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= -github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= -github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/dtls/v3 v3.0.11 h1:zqn8YhoAU7d9whsWLhNiQlbB8QdpJj8XQVSc5ImUons= -github.com/pion/dtls/v3 v3.0.11/go.mod h1:YEmmBYIoBsY3jmG56dsziTv/Lca9y4Om83370CXfqJ8= -github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4= -github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= -github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= -github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= -github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= -github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= -github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= -github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= -github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= -github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= -github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= -github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= -github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= -github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= -github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= -github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4= -github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= -github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= -github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= -github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= -github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= -github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= -github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= -github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= -github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= -github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= -github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= -github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o= -github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM= -github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= -github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= -github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= -github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -410,26 +274,18 @@ github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4 github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= -github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= -github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= -github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= -github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= -github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -444,14 +300,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -469,35 +319,24 @@ github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/S github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.3.1 h1:82ioxmhEYut7LBVGhGq8xoRkXPLElVuh5mV67AFfdv0= github.com/whyrusleeping/cbor-gen v0.3.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= -github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= -github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= -github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q= gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -506,14 +345,8 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= -go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= -go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= -go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= -go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -527,187 +360,59 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= -golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -716,39 +421,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -761,18 +433,10 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/aggregateeventrecorder/aggregateeventrecorder.go b/pkg/aggregateeventrecorder/aggregateeventrecorder.go index 1d25a930..a9016bed 100644 --- a/pkg/aggregateeventrecorder/aggregateeventrecorder.go +++ b/pkg/aggregateeventrecorder/aggregateeventrecorder.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/lassie/pkg/events" "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multicodec" ) var logger = log.Logger("lassie/aggregateeventrecorder") @@ -202,8 +201,7 @@ func (a *aggregateEventRecorder) ingestEvents() { } case events.BlockReceivedEvent: - // data received is unique in that it always has a provider - spid := ret.ProviderId().String() + spid := events.Identifier(ret) attempt, ok := tempData.retrievalAttempts[spid] if !ok { attempt = new(RetrievalAttempt) @@ -211,12 +209,6 @@ func (a *aggregateEventRecorder) ingestEvents() { tempData.retrievalAttempts[spid] = attempt } attempt.BytesTransferred += ret.ByteCount() - if ret.Protocol() == multicodec.TransportBitswap { - // record the total under the bitswap identifier as well - if _, ok := tempData.retrievalAttempts[types.BitswapIndentifier]; ok { - tempData.retrievalAttempts[types.BitswapIndentifier].BytesTransferred += ret.ByteCount() - } - } case events.FailedRetrievalEvent: // Add an error message to the retrieval attempt diff --git a/pkg/aggregateeventrecorder/aggregateeventrecorder_test.go b/pkg/aggregateeventrecorder/aggregateeventrecorder_test.go index bfb88b91..03e12c35 100644 --- a/pkg/aggregateeventrecorder/aggregateeventrecorder_test.go +++ b/pkg/aggregateeventrecorder/aggregateeventrecorder_test.go @@ -18,7 +18,6 @@ import ( "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/require" ) @@ -44,7 +43,7 @@ func TestAggregateEventRecorder(t *testing.T) { defer ts.Close() testCid1 := testutil.GenerateCid() - bitswapCandidates := testutil.GenerateRetrievalCandidatesForCID(t, 3, testCid1, &metadata.Bitswap{}) + httpCandidates := testutil.GenerateRetrievalCandidatesForCID(t, 3, testCid1, &metadata.IpfsGatewayHttp{}) graphsyncCandidates := testutil.GenerateRetrievalCandidatesForCID(t, 3, testCid1, &metadata.GraphsyncFilecoinV1{}) tests := []struct { @@ -56,7 +55,7 @@ func TestAggregateEventRecorder(t *testing.T) { exec: func(t *testing.T, ctx context.Context, subscriber types.RetrievalEventSubscriber, id types.RetrievalID) { clock := clock.NewMock() fetchStartTime := clock.Now() - subscriber(events.StartedFetch(clock.Now(), id, testCid1, "/applesauce", multicodec.TransportGraphsyncFilecoinv1, multicodec.TransportBitswap)) + subscriber(events.StartedFetch(clock.Now(), id, testCid1, "/applesauce", multicodec.TransportGraphsyncFilecoinv1, multicodec.TransportIpfsGatewayHttp)) clock.Add(10 * time.Millisecond) subscriber(events.StartedFindingCandidates(clock.Now(), id, testCid1)) subscriber(events.CandidatesFound(clock.Now(), id, testCid1, graphsyncCandidates)) @@ -64,14 +63,14 @@ func TestAggregateEventRecorder(t *testing.T) { subscriber(events.StartedRetrieval(clock.Now(), id, graphsyncCandidates[0], multicodec.TransportGraphsyncFilecoinv1)) subscriber(events.StartedRetrieval(clock.Now(), id, graphsyncCandidates[1], multicodec.TransportGraphsyncFilecoinv1)) clock.Add(10 * time.Millisecond) - subscriber(events.CandidatesFound(clock.Now(), id, testCid1, bitswapCandidates)) - subscriber(events.CandidatesFiltered(clock.Now(), id, testCid1, bitswapCandidates[:2])) - bitswapPeer := types.NewRetrievalCandidate(peer.ID(""), nil, testCid1, &metadata.Bitswap{}) - subscriber(events.StartedRetrieval(clock.Now(), id, bitswapPeer, multicodec.TransportBitswap)) + subscriber(events.CandidatesFound(clock.Now(), id, testCid1, httpCandidates)) + subscriber(events.CandidatesFiltered(clock.Now(), id, testCid1, httpCandidates[:2])) + httpPeer := httpCandidates[0] + subscriber(events.StartedRetrieval(clock.Now(), id, httpPeer, multicodec.TransportIpfsGatewayHttp)) clock.Add(20 * time.Millisecond) - subscriber(events.FirstByte(clock.Now(), id, bitswapPeer, 20*time.Millisecond, multicodec.TransportBitswap)) - subscriber(events.BlockReceived(clock.Now(), id, bitswapCandidates[0], multicodec.TransportBitswap, 3000)) - subscriber(events.BlockReceived(clock.Now(), id, bitswapCandidates[0], multicodec.TransportBitswap, 2000)) + subscriber(events.FirstByte(clock.Now(), id, httpPeer, 20*time.Millisecond, multicodec.TransportIpfsGatewayHttp)) + subscriber(events.BlockReceived(clock.Now(), id, httpCandidates[0], multicodec.TransportIpfsGatewayHttp, 3000)) + subscriber(events.BlockReceived(clock.Now(), id, httpCandidates[0], multicodec.TransportIpfsGatewayHttp, 2000)) subscriber(events.FailedRetrieval(clock.Now(), id, graphsyncCandidates[0], multicodec.TransportGraphsyncFilecoinv1, "failed to dial")) clock.Add(20 * time.Millisecond) subscriber(events.FirstByte(clock.Now(), id, graphsyncCandidates[1], 50*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1)) @@ -79,9 +78,9 @@ func TestAggregateEventRecorder(t *testing.T) { subscriber(events.BlockReceived(clock.Now(), id, graphsyncCandidates[1], multicodec.TransportGraphsyncFilecoinv1, 3000)) subscriber(events.BlockReceived(clock.Now(), id, graphsyncCandidates[1], multicodec.TransportGraphsyncFilecoinv1, 2000)) - subscriber(events.BlockReceived(clock.Now(), id, bitswapCandidates[1], multicodec.TransportBitswap, 5000)) - subscriber(events.Success(clock.Now(), id, bitswapPeer, uint64(10000), 3030, 4*time.Second, multicodec.TransportBitswap)) - subscriber(events.Finished(clock.Now(), id, bitswapPeer)) + subscriber(events.BlockReceived(clock.Now(), id, httpCandidates[1], multicodec.TransportIpfsGatewayHttp, 5000)) + subscriber(events.Success(clock.Now(), id, httpPeer, uint64(10000), 3030, 4*time.Second, multicodec.TransportIpfsGatewayHttp)) + subscriber(events.Finished(clock.Now(), id, httpPeer)) var req gotReq select { @@ -100,7 +99,7 @@ func TestAggregateEventRecorder(t *testing.T) { verifyStringNode(t, event, "retrievalId", id.String()) verifyStringNode(t, event, "rootCid", testCid1.String()) verifyStringNode(t, event, "urlPath", "/applesauce") - verifyStringNode(t, event, "storageProviderId", types.BitswapIndentifier) + verifyStringNode(t, event, "storageProviderId", httpPeer.Endpoint()) verifyStringNode(t, event, "timeToFirstByte", "40ms") verifyStringNode(t, event, "timeToFirstIndexerResult", "10ms") verifyIntNode(t, event, "bandwidth", 200000) @@ -111,40 +110,35 @@ func TestAggregateEventRecorder(t *testing.T) { verifyIntNode(t, event, "indexerCandidatesReceived", 6) verifyIntNode(t, event, "indexerCandidatesFiltered", 4) protocolsAllowed := verifyListNode(t, event, "protocolsAllowed", 2) - verifyStringListElementsMatch(t, protocolsAllowed, []string{"transport-graphsync-filecoinv1", "transport-bitswap"}) + verifyStringListElementsMatch(t, protocolsAllowed, []string{"transport-graphsync-filecoinv1", "transport-ipfs-gateway-http"}) protocolsAttempted := verifyListNode(t, event, "protocolsAttempted", 2) - verifyStringListElementsMatch(t, protocolsAttempted, []string{"transport-graphsync-filecoinv1", "transport-bitswap"}) - verifyStringNode(t, event, "protocolSucceeded", "transport-bitswap") + verifyStringListElementsMatch(t, protocolsAttempted, []string{"transport-graphsync-filecoinv1", "transport-ipfs-gateway-http"}) + verifyStringNode(t, event, "protocolSucceeded", "transport-ipfs-gateway-http") retrievalAttempts, err := event.LookupByString("retrievalAttempts") require.NoError(t, err) - require.Equal(t, int64(5), retrievalAttempts.Length()) - sp1Attempt, err := retrievalAttempts.LookupByString(graphsyncCandidates[0].MinerPeer.ID.String()) + require.Equal(t, int64(4), retrievalAttempts.Length()) + sp1Attempt, err := retrievalAttempts.LookupByString(graphsyncCandidates[0].Endpoint()) require.NoError(t, err) require.Equal(t, int64(2), sp1Attempt.Length()) verifyStringNode(t, sp1Attempt, "protocol", multicodec.TransportGraphsyncFilecoinv1.String()) verifyStringNode(t, sp1Attempt, "error", "failed to dial") - sp2Attempt, err := retrievalAttempts.LookupByString(graphsyncCandidates[1].MinerPeer.ID.String()) + sp2Attempt, err := retrievalAttempts.LookupByString(graphsyncCandidates[1].Endpoint()) require.NoError(t, err) require.Equal(t, int64(3), sp2Attempt.Length()) verifyStringNode(t, sp2Attempt, "protocol", multicodec.TransportGraphsyncFilecoinv1.String()) verifyStringNode(t, sp2Attempt, "timeToFirstByte", "50ms") verifyIntNode(t, sp2Attempt, "bytesTransferred", 5000) - bitswapAttempt, err := retrievalAttempts.LookupByString(types.BitswapIndentifier) + httpPeer1Attempt, err := retrievalAttempts.LookupByString(httpCandidates[0].Endpoint()) require.NoError(t, err) - require.Equal(t, int64(3), bitswapAttempt.Length()) - verifyStringNode(t, bitswapAttempt, "timeToFirstByte", "20ms") - verifyStringNode(t, bitswapAttempt, "protocol", multicodec.TransportBitswap.String()) - verifyIntNode(t, bitswapAttempt, "bytesTransferred", 10000) - bitswapPeer1Attempt, err := retrievalAttempts.LookupByString(bitswapCandidates[0].MinerPeer.ID.String()) + require.Equal(t, int64(3), httpPeer1Attempt.Length()) + verifyStringNode(t, httpPeer1Attempt, "timeToFirstByte", "20ms") + verifyStringNode(t, httpPeer1Attempt, "protocol", multicodec.TransportIpfsGatewayHttp.String()) + verifyIntNode(t, httpPeer1Attempt, "bytesTransferred", 5000) + httpPeer2Attempt, err := retrievalAttempts.LookupByString(httpCandidates[1].Endpoint()) require.NoError(t, err) - require.Equal(t, int64(2), bitswapPeer1Attempt.Length()) - verifyIntNode(t, bitswapPeer1Attempt, "bytesTransferred", 5000) - verifyStringNode(t, bitswapPeer1Attempt, "protocol", multicodec.TransportBitswap.String()) - bitswapPeer2Attempt, err := retrievalAttempts.LookupByString(bitswapCandidates[1].MinerPeer.ID.String()) - require.NoError(t, err) - require.Equal(t, int64(2), bitswapPeer2Attempt.Length()) - verifyIntNode(t, bitswapPeer2Attempt, "bytesTransferred", 5000) - verifyStringNode(t, bitswapPeer2Attempt, "protocol", multicodec.TransportBitswap.String()) + require.Equal(t, int64(2), httpPeer2Attempt.Length()) + verifyIntNode(t, httpPeer2Attempt, "bytesTransferred", 5000) + verifyStringNode(t, httpPeer2Attempt, "protocol", multicodec.TransportIpfsGatewayHttp.String()) }, }, { diff --git a/pkg/blockbroker/session.go b/pkg/blockbroker/session.go new file mode 100644 index 00000000..ac24d4f4 --- /dev/null +++ b/pkg/blockbroker/session.go @@ -0,0 +1,435 @@ +package blockbroker + +import ( + "context" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/filecoin-project/lassie/pkg/types" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-log/v2" + "github.com/ipld/go-ipld-prime/linking" + trustlessutils "github.com/ipld/go-trustless-utils" + trustlesshttp "github.com/ipld/go-trustless-utils/http" + "github.com/ipld/go-trustless-utils/traversal" + "github.com/libp2p/go-libp2p/core/peer" +) + +var logger = log.Logger("lassie/blockbroker") + +var _ BlockSession = (*TrustlessGatewaySession)(nil) + +// TrustlessGatewaySession fetches blocks from HTTP trustless gateways, +// caching discovered providers and evicting ones that fail. +type TrustlessGatewaySession struct { + routing types.CandidateSource + httpClient *http.Client + + providers []types.RetrievalCandidate + providersLock sync.RWMutex + + evictedPeers map[peer.ID]time.Time + evictedPeersLock sync.RWMutex + evictBackoff time.Duration + + usedProviders map[string]struct{} + usedProvidersLock sync.RWMutex + + skipBlockVerification bool +} + +func NewSession( + routing types.CandidateSource, + httpClient *http.Client, + skipBlockVerification bool, +) *TrustlessGatewaySession { + if httpClient == nil { + httpClient = http.DefaultClient + } + + return &TrustlessGatewaySession{ + routing: routing, + httpClient: httpClient, + providers: make([]types.RetrievalCandidate, 0), + evictedPeers: make(map[peer.ID]time.Time), + evictBackoff: 30 * time.Second, + usedProviders: make(map[string]struct{}), + skipBlockVerification: skipBlockVerification, + } +} + +func (s *TrustlessGatewaySession) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + providers := s.getProviders() + if len(providers) > 0 { + block, err := s.tryProviders(ctx, c, providers) + if err == nil { + return block, nil + } + logger.Debugw("all cached providers failed", "cid", c, "err", err) + } + + if err := s.findNewProviders(ctx, c); err != nil { + return nil, fmt.Errorf("no providers found for %s: %w", c, err) + } + + providers = s.getProviders() + if len(providers) == 0 { + return nil, fmt.Errorf("no providers available for %s", c) + } + return s.tryProviders(ctx, c, providers) +} + +func (s *TrustlessGatewaySession) GetSubgraph(ctx context.Context, c cid.Cid, lsys linking.LinkSystem) (int, error) { + providers := s.getProviders() + if len(providers) == 0 { + if err := s.findNewProviders(ctx, c); err != nil { + return 0, fmt.Errorf("no providers found for %s: %w", c, err) + } + providers = s.getProviders() + } + + if len(providers) == 0 { + return 0, fmt.Errorf("no providers available for %s", c) + } + + var lastErr error + for _, provider := range providers { + blocksReceived, err := s.fetchSubgraphCAR(ctx, c, provider, lsys) + if err == nil { + logger.Debugw("subgraph CAR fetch succeeded", "cid", c, "provider", provider.Endpoint(), "blocks", blocksReceived) + s.markProviderUsed(provider.Endpoint()) + return blocksReceived, nil + } + logger.Debugw("subgraph CAR fetch failed", "cid", c, "provider", provider.Endpoint(), "err", err) + lastErr = err + } + + return 0, fmt.Errorf("all providers failed for subgraph %s: %w", c, lastErr) +} + +func (s *TrustlessGatewaySession) fetchSubgraphCAR( + ctx context.Context, + c cid.Cid, + provider types.RetrievalCandidate, + lsys linking.LinkSystem, +) (int, error) { + providerURL, err := provider.ToURL() + if err != nil { + return 0, fmt.Errorf("failed to get URL for provider: %w", err) + } + + reqURL := fmt.Sprintf("%s/ipfs/%s?dag-scope=all", providerURL, c) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return 0, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Accept", trustlesshttp.DefaultContentType().String()) + + resp, err := s.httpClient.Do(req) + if err != nil { + return 0, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("HTTP %d from %s", resp.StatusCode, providerURL) + } + + expectDuplicates := trustlesshttp.DefaultIncludeDupes + if contentType, valid := trustlesshttp.ParseContentType(resp.Header.Get("Content-Type")); valid { + expectDuplicates = contentType.Duplicates + } + + defaultSelector := trustlessutils.Request{Scope: trustlessutils.DagScopeAll}.Selector() + cfg := traversal.Config{ + Root: c, + Selector: defaultSelector, + ExpectDuplicatesIn: expectDuplicates, + WriteDuplicatesOut: expectDuplicates, + } + + result, err := cfg.VerifyCar(ctx, resp.Body, lsys) + if err != nil { + return int(result.BlocksIn), err + } + + return int(result.BlocksIn), nil +} + +func (s *TrustlessGatewaySession) getProviders() []types.RetrievalCandidate { + s.providersLock.RLock() + defer s.providersLock.RUnlock() + result := make([]types.RetrievalCandidate, len(s.providers)) + copy(result, s.providers) + return result +} + +func (s *TrustlessGatewaySession) tryProviders(ctx context.Context, c cid.Cid, providers []types.RetrievalCandidate) (blocks.Block, error) { + if len(providers) == 0 { + return nil, fmt.Errorf("no providers available") + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + type fetchResult struct { + block blocks.Block + err error + provider types.RetrievalCandidate + } + + results := make(chan fetchResult, len(providers)) + for _, p := range providers { + go func(provider types.RetrievalCandidate) { + block, err := s.fetchBlock(ctx, c, provider) + select { + case results <- fetchResult{block: block, err: err, provider: provider}: + case <-ctx.Done(): + } + }(p) + } + + var lastErr error + for range providers { + select { + case result := <-results: + if result.err != nil { + logger.Debugw("provider failed", "cid", c, "provider", result.provider.Endpoint(), "err", result.err) + s.evictProvider(result.provider.MinerPeer.ID) + lastErr = result.err + continue + } + cancel() + s.markProviderUsed(result.provider.Endpoint()) + return result.block, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return nil, fmt.Errorf("all providers failed: %w", lastErr) +} + +func (s *TrustlessGatewaySession) fetchBlock( + ctx context.Context, + c cid.Cid, + provider types.RetrievalCandidate, +) (blocks.Block, error) { + url, err := provider.ToURL() + if err != nil { + return nil, fmt.Errorf("failed to get URL for provider: %w", err) + } + + reqURL := fmt.Sprintf("%s/ipfs/%s?format=raw", url, c) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Accept", "application/vnd.ipld.raw") + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d from %s", resp.StatusCode, url) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + if !s.skipBlockVerification { + computed, err := c.Prefix().Sum(data) + if err != nil { + return nil, fmt.Errorf("failed to compute hash: %w", err) + } + if !computed.Equals(c) { + return nil, fmt.Errorf("cid mismatch: expected %s, got %s", c, computed) + } + } + + block, err := blocks.NewBlockWithCid(data, c) + if err != nil { + return nil, fmt.Errorf("block creation failed: %w", err) + } + + return block, nil +} + +func (s *TrustlessGatewaySession) findNewProviders(ctx context.Context, c cid.Cid) error { + var found int + var mu sync.Mutex + + err := s.routing.FindCandidates(ctx, c, func(candidate types.RetrievalCandidate) { + if !s.hasHTTPProtocol(candidate) { + return + } + if s.isEvicted(candidate.MinerPeer.ID) { + return + } + + mu.Lock() + s.addProvider(candidate) + found++ + mu.Unlock() + }) + + if err != nil { + return err + } + if found == 0 { + return fmt.Errorf("no HTTP providers found for %s", c) + } + logger.Debugw("found new providers", "cid", c, "count", found) + return nil +} + +func (s *TrustlessGatewaySession) SeedProviders(ctx context.Context, c cid.Cid) { + _ = s.findNewProviders(ctx, c) +} + +func (s *TrustlessGatewaySession) addProvider(candidate types.RetrievalCandidate) { + s.providersLock.Lock() + defer s.providersLock.Unlock() + + for _, p := range s.providers { + if p.MinerPeer.ID == candidate.MinerPeer.ID { + return + } + } + s.providers = append(s.providers, candidate) +} + +func (s *TrustlessGatewaySession) evictProvider(peerID peer.ID) { + s.evictedPeersLock.Lock() + s.evictedPeers[peerID] = time.Now() + s.evictedPeersLock.Unlock() + + s.providersLock.Lock() + defer s.providersLock.Unlock() + + for i, p := range s.providers { + if p.MinerPeer.ID == peerID { + s.providers = append(s.providers[:i], s.providers[i+1:]...) + return + } + } +} + +func (s *TrustlessGatewaySession) isEvicted(peerID peer.ID) bool { + s.evictedPeersLock.RLock() + defer s.evictedPeersLock.RUnlock() + + evictTime, ok := s.evictedPeers[peerID] + if !ok { + return false + } + return time.Since(evictTime) < s.evictBackoff +} + +func (s *TrustlessGatewaySession) hasHTTPProtocol(candidate types.RetrievalCandidate) bool { + for _, addr := range candidate.MinerPeer.Addrs { + for _, proto := range addr.Protocols() { + if proto.Name == "http" || proto.Name == "https" { + return true + } + } + } + return false +} + +func (s *TrustlessGatewaySession) markProviderUsed(endpoint string) { + s.usedProvidersLock.Lock() + defer s.usedProvidersLock.Unlock() + s.usedProviders[endpoint] = struct{}{} +} + +func (s *TrustlessGatewaySession) UsedProviders() []string { + s.usedProvidersLock.RLock() + defer s.usedProvidersLock.RUnlock() + result := make([]string, 0, len(s.usedProviders)) + for endpoint := range s.usedProviders { + result = append(result, endpoint) + } + return result +} + +func (s *TrustlessGatewaySession) GetSubgraphStream(ctx context.Context, c cid.Cid) (io.ReadCloser, string, error) { + providers := s.getProviders() + if len(providers) == 0 { + if err := s.findNewProviders(ctx, c); err != nil { + return nil, "", fmt.Errorf("no providers found for %s: %w", c, err) + } + providers = s.getProviders() + } + + if len(providers) == 0 { + return nil, "", fmt.Errorf("no providers available for %s", c) + } + + var lastErr error + for _, provider := range providers { + rdr, err := s.fetchSubgraphStream(ctx, c, provider) + if err == nil { + s.markProviderUsed(provider.Endpoint()) + return rdr, provider.Endpoint(), nil + } + logger.Debugw("subgraph stream fetch failed", "cid", c, "provider", provider.Endpoint(), "err", err) + lastErr = err + } + + return nil, "", fmt.Errorf("all providers failed for subgraph stream %s: %w", c, lastErr) +} + +func (s *TrustlessGatewaySession) fetchSubgraphStream( + ctx context.Context, + c cid.Cid, + provider types.RetrievalCandidate, +) (io.ReadCloser, error) { + providerURL, err := provider.ToURL() + if err != nil { + return nil, fmt.Errorf("failed to get URL for provider: %w", err) + } + + reqURL := fmt.Sprintf("%s/ipfs/%s?dag-scope=all&dups=y", providerURL, c) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Accept", trustlesshttp.DefaultContentType().String()) + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("HTTP %d from %s", resp.StatusCode, providerURL) + } + + return resp.Body, nil +} + +func (s *TrustlessGatewaySession) Close() error { + s.providersLock.Lock() + s.providers = nil + s.providersLock.Unlock() + + s.evictedPeersLock.Lock() + s.evictedPeers = nil + s.evictedPeersLock.Unlock() + + s.usedProvidersLock.Lock() + s.usedProviders = nil + s.usedProvidersLock.Unlock() + + return nil +} diff --git a/pkg/blockbroker/session_test.go b/pkg/blockbroker/session_test.go new file mode 100644 index 00000000..ee76ffb2 --- /dev/null +++ b/pkg/blockbroker/session_test.go @@ -0,0 +1,115 @@ +package blockbroker + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + "github.com/ipni/go-libipni/metadata" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +// tests both bitrot and malicious content detection by flipping a single bit +func TestCIDMismatchRejection(t *testing.T) { + ctx := context.Background() + + correctData := []byte("this is valid block content that will be corrupted") + mh, err := multihash.Sum(correctData, multihash.SHA2_256, -1) + require.NoError(t, err) + c := cid.NewCidV1(cid.Raw, mh) + + // corrupt by flipping one bit (simulates bitrot or subtle malicious modification) + corruptedData := make([]byte, len(correctData)) + copy(corruptedData, correctData) + corruptedData[len(corruptedData)/2] ^= 0x01 // flip lowest bit in middle byte + + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Content-Type", "application/vnd.ipld.raw") + w.WriteHeader(http.StatusOK) + w.Write(corruptedData) + })) + defer server.Close() + + provider := makeTestCandidate(t, server.URL, c) + session := NewSession(&mockRouting{providers: []types.RetrievalCandidate{provider}}, http.DefaultClient, false) + defer session.Close() + + session.addProvider(provider) + + block, err := session.tryProviders(ctx, c, []types.RetrievalCandidate{provider}) + + require.Nil(t, block, "corrupted block must not be returned") + require.Error(t, err, "single bit flip must be detected") + require.Contains(t, err.Error(), "cid mismatch") + require.Equal(t, 1, requestCount) +} + +func TestCorrectBlockAccepted(t *testing.T) { + ctx := context.Background() + + testData := []byte("hello world") + mh, err := multihash.Sum(testData, multihash.SHA2_256, -1) + require.NoError(t, err) + c := cid.NewCidV1(cid.Raw, mh) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/vnd.ipld.raw") + w.WriteHeader(http.StatusOK) + w.Write(testData) + })) + defer server.Close() + + provider := makeTestCandidate(t, server.URL, c) + session := NewSession(&mockRouting{providers: []types.RetrievalCandidate{provider}}, http.DefaultClient, false) + defer session.Close() + + session.SeedProviders(ctx, c) + + block, err := session.Get(ctx, c) + require.NoError(t, err) + require.Equal(t, testData, block.RawData()) + require.Equal(t, c, block.Cid()) +} + +type mockRouting struct { + providers []types.RetrievalCandidate +} + +func (m *mockRouting) FindCandidates(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error { + for _, p := range m.providers { + cb(p) + } + return nil +} + +func makeTestCandidate(t *testing.T, serverURL string, rootCid cid.Cid) types.RetrievalCandidate { + // convert http://host:port to /ip4/host/tcp/port/http multiaddr + u, err := url.Parse(serverURL) + require.NoError(t, err) + host, port, err := net.SplitHostPort(u.Host) + require.NoError(t, err) + + maddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s/http", host, port)) + require.NoError(t, err) + + pid, err := peer.Decode("12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4") + require.NoError(t, err) + + return types.NewRetrievalCandidate( + pid, + []multiaddr.Multiaddr{maddr}, + rootCid, + &metadata.IpfsGatewayHttp{}, + ) +} diff --git a/pkg/blockbroker/types.go b/pkg/blockbroker/types.go new file mode 100644 index 00000000..dc79a55f --- /dev/null +++ b/pkg/blockbroker/types.go @@ -0,0 +1,19 @@ +package blockbroker + +import ( + "context" + "io" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/linking" +) + +type BlockSession interface { + Get(ctx context.Context, c cid.Cid) (blocks.Block, error) + GetSubgraph(ctx context.Context, c cid.Cid, lsys linking.LinkSystem) (int, error) + GetSubgraphStream(ctx context.Context, c cid.Cid) (io.ReadCloser, string, error) // caller must close + SeedProviders(ctx context.Context, c cid.Cid) + UsedProviders() []string + Close() error +} diff --git a/pkg/events/base.go b/pkg/events/base.go index f18072a2..0f2bf59a 100644 --- a/pkg/events/base.go +++ b/pkg/events/base.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" - "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multicodec" ) @@ -21,14 +20,14 @@ func (r retrievalEvent) RootCid() cid.Cid { return r.rootCid } type providerRetrievalEvent struct { retrievalEvent - providerId peer.ID + endpoint string } -func (e providerRetrievalEvent) ProviderId() peer.ID { return e.providerId } +func (e providerRetrievalEvent) Endpoint() string { return e.endpoint } -type EventWithProviderID interface { +type EventWithEndpoint interface { types.RetrievalEvent - ProviderId() peer.ID + Endpoint() string } type EventWithCandidates interface { diff --git a/pkg/events/blockreceived.go b/pkg/events/blockreceived.go index 5b35fbcc..ce0fd20c 100644 --- a/pkg/events/blockreceived.go +++ b/pkg/events/blockreceived.go @@ -11,7 +11,7 @@ import ( var ( _ types.RetrievalEvent = BlockReceivedEvent{} _ EventWithProtocol = BlockReceivedEvent{} - _ EventWithProviderID = BlockReceivedEvent{} + _ EventWithEndpoint = BlockReceivedEvent{} ) // BlockReceivedEvent records new data received from a provider. It is used to track how much is downloaded from each peer in a retrieval @@ -25,9 +25,9 @@ func (e BlockReceivedEvent) Code() types.EventCode { return types.BlockRecei func (e BlockReceivedEvent) ByteCount() uint64 { return e.byteCount } func (e BlockReceivedEvent) Protocol() multicodec.Code { return e.protocol } func (e BlockReceivedEvent) String() string { - return fmt.Sprintf("BlockReceivedEvent<%s, %s, %s, %s, %s, %d>", e.eventTime, e.retrievalId, e.rootCid, e.providerId, e.protocol, e.byteCount) + return fmt.Sprintf("BlockReceivedEvent<%s, %s, %s, %s, %s, %d>", e.eventTime, e.retrievalId, e.rootCid, e.endpoint, e.protocol, e.byteCount) } func BlockReceived(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate, protocol multicodec.Code, byteCount uint64) BlockReceivedEvent { - return BlockReceivedEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}, protocol, byteCount} + return BlockReceivedEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.Endpoint()}, protocol, byteCount} } diff --git a/pkg/events/connectedtoprovider.go b/pkg/events/connectedtoprovider.go index 4d95f9ad..ad36aa28 100644 --- a/pkg/events/connectedtoprovider.go +++ b/pkg/events/connectedtoprovider.go @@ -10,7 +10,7 @@ import ( var ( _ types.RetrievalEvent = ConnectedToProviderEvent{} - _ EventWithProviderID = ConnectedToProviderEvent{} + _ EventWithEndpoint = ConnectedToProviderEvent{} ) type ConnectedToProviderEvent struct { @@ -21,9 +21,9 @@ type ConnectedToProviderEvent struct { func (e ConnectedToProviderEvent) Code() types.EventCode { return types.ConnectedToProviderCode } func (e ConnectedToProviderEvent) Protocol() multicodec.Code { return e.protocol } func (e ConnectedToProviderEvent) String() string { - return fmt.Sprintf("ConnectedToProviderEvent<%s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.providerId) + return fmt.Sprintf("ConnectedToProviderEvent<%s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.endpoint) } func ConnectedToProvider(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate, protocol multicodec.Code) ConnectedToProviderEvent { - return ConnectedToProviderEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}, protocol} + return ConnectedToProviderEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.Endpoint()}, protocol} } diff --git a/pkg/events/events.go b/pkg/events/events.go index e6603886..1ee5762c 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -2,22 +2,13 @@ package events import ( "github.com/filecoin-project/lassie/pkg/types" - "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multicodec" ) -// Identifier returns the peer ID of the storage provider if this retrieval was -// requested via peer ID, or the string "Bitswap" if this retrieval was -// requested via the Bitswap protocol +// Identifier returns the HTTP endpoint of the storage provider func Identifier(evt types.RetrievalEvent) string { - spEvent, spOk := evt.(EventWithProviderID) - if spOk && spEvent.ProviderId() != peer.ID("") { - return spEvent.ProviderId().String() - } - // we only want to return "Bitswap" if this is an event with a storage provider id using the bitswap protocol - protocolEvent, pOk := evt.(EventWithProtocol) - if spOk && pOk && protocolEvent.Protocol() == multicodec.TransportBitswap { - return types.BitswapIndentifier + if epEvent, ok := evt.(EventWithEndpoint); ok { + return epEvent.Endpoint() } return "" } diff --git a/pkg/events/extraction.go b/pkg/events/extraction.go new file mode 100644 index 00000000..22a20b65 --- /dev/null +++ b/pkg/events/extraction.go @@ -0,0 +1,57 @@ +package events + +import ( + "fmt" + "time" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" +) + +var ( + _ types.RetrievalEvent = ExtractionStartedEvent{} + _ EventWithEndpoint = ExtractionStartedEvent{} +) + +type ExtractionStartedEvent struct { + providerRetrievalEvent +} + +func (e ExtractionStartedEvent) Code() types.EventCode { return types.StartedRetrievalCode } +func (e ExtractionStartedEvent) String() string { + return fmt.Sprintf("ExtractionStartedEvent<%s, %s, %s>", e.eventTime, e.rootCid, e.endpoint) +} + +func ExtractionStarted(at time.Time, rootCid cid.Cid, endpoint string) ExtractionStartedEvent { + return ExtractionStartedEvent{providerRetrievalEvent{retrievalEvent{at, types.RetrievalID{}, rootCid}, endpoint}} +} + +var ( + _ types.RetrievalEvent = ExtractionSucceededEvent{} + _ EventWithEndpoint = ExtractionSucceededEvent{} +) + +type ExtractionSucceededEvent struct { + providerRetrievalEvent + bytesExtracted uint64 + blocksExtracted uint64 + duration time.Duration +} + +func (e ExtractionSucceededEvent) Code() types.EventCode { return types.SuccessCode } +func (e ExtractionSucceededEvent) BytesExtracted() uint64 { return e.bytesExtracted } +func (e ExtractionSucceededEvent) BlocksExtracted() uint64 { return e.blocksExtracted } +func (e ExtractionSucceededEvent) Duration() time.Duration { return e.duration } +func (e ExtractionSucceededEvent) String() string { + return fmt.Sprintf("ExtractionSucceededEvent<%s, %s, %s, %d bytes, %d blocks, %s>", + e.eventTime, e.rootCid, e.endpoint, e.bytesExtracted, e.blocksExtracted, e.duration) +} + +func ExtractionSucceeded(at time.Time, rootCid cid.Cid, endpoint string, bytes, blocks uint64, duration time.Duration) ExtractionSucceededEvent { + return ExtractionSucceededEvent{ + providerRetrievalEvent: providerRetrievalEvent{retrievalEvent{at, types.RetrievalID{}, rootCid}, endpoint}, + bytesExtracted: bytes, + blocksExtracted: blocks, + duration: duration, + } +} diff --git a/pkg/events/failedretrieval.go b/pkg/events/failedretrieval.go index 0e079cc9..3a47c8b0 100644 --- a/pkg/events/failedretrieval.go +++ b/pkg/events/failedretrieval.go @@ -10,7 +10,7 @@ import ( var ( _ types.RetrievalEvent = FailedRetrievalEvent{} - _ EventWithProviderID = FailedRetrievalEvent{} + _ EventWithEndpoint = FailedRetrievalEvent{} _ EventWithErrorMessage = FailedRetrievalEvent{} ) @@ -28,5 +28,5 @@ func (e FailedRetrievalEvent) String() string { } func FailedRetrieval(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate, protocol multicodec.Code, errorMessage string) FailedRetrievalEvent { - return FailedRetrievalEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}, protocol, errorMessage} + return FailedRetrievalEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.Endpoint()}, protocol, errorMessage} } diff --git a/pkg/events/finished.go b/pkg/events/finished.go index 0d179cfd..04cc0a1b 100644 --- a/pkg/events/finished.go +++ b/pkg/events/finished.go @@ -9,7 +9,7 @@ import ( var ( _ types.RetrievalEvent = FinishedEvent{} - _ EventWithProviderID = FinishedEvent{} + _ EventWithEndpoint = FinishedEvent{} ) type FinishedEvent struct { @@ -18,9 +18,9 @@ type FinishedEvent struct { func (e FinishedEvent) Code() types.EventCode { return types.FinishedCode } func (e FinishedEvent) String() string { - return fmt.Sprintf("FinishedEvent<%s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.providerId) + return fmt.Sprintf("FinishedEvent<%s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.endpoint) } func Finished(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate) FinishedEvent { - return FinishedEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}} + return FinishedEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.Endpoint()}} } diff --git a/pkg/events/firstbyte.go b/pkg/events/firstbyte.go index 370a81a1..4e951a07 100644 --- a/pkg/events/firstbyte.go +++ b/pkg/events/firstbyte.go @@ -10,7 +10,7 @@ import ( var ( _ types.RetrievalEvent = FirstByteEvent{} - _ EventWithProviderID = FirstByteEvent{} + _ EventWithEndpoint = FirstByteEvent{} _ EventWithProtocol = FirstByteEvent{} ) @@ -24,9 +24,9 @@ func (e FirstByteEvent) Code() types.EventCode { return types.FirstByteCode func (e FirstByteEvent) Duration() time.Duration { return e.duration } func (e FirstByteEvent) Protocol() multicodec.Code { return e.protocol } func (e FirstByteEvent) String() string { - return fmt.Sprintf("FirstByteEvent<%s, %s, %s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.providerId, e.duration.String(), e.protocol.String()) + return fmt.Sprintf("FirstByteEvent<%s, %s, %s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.endpoint, e.duration.String(), e.protocol.String()) } func FirstByte(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate, duration time.Duration, protocol multicodec.Code) FirstByteEvent { - return FirstByteEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}, duration, protocol} + return FirstByteEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.Endpoint()}, duration, protocol} } diff --git a/pkg/events/graphsyncaccepted.go b/pkg/events/graphsyncaccepted.go deleted file mode 100644 index b80336d6..00000000 --- a/pkg/events/graphsyncaccepted.go +++ /dev/null @@ -1,30 +0,0 @@ -package events - -import ( - "fmt" - "time" - - "github.com/filecoin-project/lassie/pkg/types" - "github.com/multiformats/go-multicodec" -) - -var ( - _ types.RetrievalEvent = GraphsyncAcceptedEvent{} - _ EventWithProviderID = GraphsyncAcceptedEvent{} -) - -type GraphsyncAcceptedEvent struct { - providerRetrievalEvent -} - -func (e GraphsyncAcceptedEvent) Code() types.EventCode { return types.AcceptedCode } -func (e GraphsyncAcceptedEvent) Protocol() multicodec.Code { - return multicodec.TransportGraphsyncFilecoinv1 -} -func (e GraphsyncAcceptedEvent) String() string { - return fmt.Sprintf("GraphsyncAcceptedEvent<%s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.providerId) -} - -func Accepted(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate) GraphsyncAcceptedEvent { - return GraphsyncAcceptedEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}} -} diff --git a/pkg/events/graphsyncproposed.go b/pkg/events/graphsyncproposed.go deleted file mode 100644 index 6890a704..00000000 --- a/pkg/events/graphsyncproposed.go +++ /dev/null @@ -1,30 +0,0 @@ -package events - -import ( - "fmt" - "time" - - "github.com/filecoin-project/lassie/pkg/types" - "github.com/multiformats/go-multicodec" -) - -var ( - _ types.RetrievalEvent = GraphsyncProposedEvent{} - _ EventWithProviderID = GraphsyncProposedEvent{} -) - -type GraphsyncProposedEvent struct { - providerRetrievalEvent -} - -func (e GraphsyncProposedEvent) Code() types.EventCode { return types.ProposedCode } -func (e GraphsyncProposedEvent) Protocol() multicodec.Code { - return multicodec.TransportGraphsyncFilecoinv1 -} -func (e GraphsyncProposedEvent) String() string { - return fmt.Sprintf("GraphsyncProposedEvent<%s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.providerId) -} - -func Proposed(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate) GraphsyncProposedEvent { - return GraphsyncProposedEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}} -} diff --git a/pkg/events/manager_test.go b/pkg/events/manager_test.go index 9baf4936..9f5c3204 100644 --- a/pkg/events/manager_test.go +++ b/pkg/events/manager_test.go @@ -120,7 +120,6 @@ func TestEventManager(t *testing.T) { require.Equal(t, id, event.RetrievalId()) require.Equal(t, cid, event.RootCid()) require.Equal(t, types.StartedRetrievalCode, event.Code()) - require.Equal(t, peerB, event.(events.StartedRetrievalEvent).ProviderId()) } verifyEvent(gotEvents1, types.StartedRetrievalCode, verifyRetrievalStarted) verifyEvent(gotEvents2, types.StartedRetrievalCode, verifyRetrievalStarted) @@ -131,7 +130,6 @@ func TestEventManager(t *testing.T) { require.Equal(t, cid, event.RootCid()) successEvent := event.(events.SucceededEvent) - require.Equal(t, peerB, successEvent.ProviderId()) require.Equal(t, uint64(100), successEvent.ReceivedBytesSize()) require.Equal(t, uint64(200), successEvent.ReceivedCidsCount()) } @@ -144,7 +142,6 @@ func TestEventManager(t *testing.T) { require.Equal(t, cid, event.RootCid()) failedEvent := event.(events.FailedRetrievalEvent) - require.Equal(t, peerB, failedEvent.ProviderId()) require.Equal(t, "error @ retrieval failure", failedEvent.ErrorMessage()) } verifyEvent(gotEvents1, types.FailedRetrievalCode, verifyRetrievalFailure) diff --git a/pkg/events/startedretrieval.go b/pkg/events/startedretrieval.go index 2eb5f9cb..78f2dd54 100644 --- a/pkg/events/startedretrieval.go +++ b/pkg/events/startedretrieval.go @@ -10,7 +10,7 @@ import ( var ( _ types.RetrievalEvent = StartedRetrievalEvent{} - _ EventWithProviderID = StartedRetrievalEvent{} + _ EventWithEndpoint = StartedRetrievalEvent{} _ EventWithProtocol = StartedRetrievalEvent{} ) @@ -23,9 +23,9 @@ type StartedRetrievalEvent struct { func (e StartedRetrievalEvent) Code() types.EventCode { return types.StartedRetrievalCode } func (e StartedRetrievalEvent) Protocol() multicodec.Code { return e.protocol } func (e StartedRetrievalEvent) String() string { - return fmt.Sprintf("StartedRetrievalEvent<%s, %s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.providerId, e.protocol) + return fmt.Sprintf("StartedRetrievalEvent<%s, %s, %s, %s, %s>", e.eventTime, e.retrievalId, e.rootCid, e.endpoint, e.protocol) } func StartedRetrieval(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate, protocol multicodec.Code) StartedRetrievalEvent { - return StartedRetrievalEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}, protocol} + return StartedRetrievalEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.Endpoint()}, protocol} } diff --git a/pkg/events/succeeded.go b/pkg/events/succeeded.go index 1d58bea3..259da7ae 100644 --- a/pkg/events/succeeded.go +++ b/pkg/events/succeeded.go @@ -10,7 +10,7 @@ import ( var ( _ types.RetrievalEvent = SucceededEvent{} - _ EventWithProviderID = SucceededEvent{} + _ EventWithEndpoint = SucceededEvent{} _ EventWithProtocol = SucceededEvent{} ) @@ -28,9 +28,9 @@ func (e SucceededEvent) ReceivedCidsCount() uint64 { return e.receivedCidsCount func (e SucceededEvent) Duration() time.Duration { return e.duration } func (e SucceededEvent) Protocol() multicodec.Code { return e.protocol } func (e SucceededEvent) String() string { - return fmt.Sprintf("SucceededEvent<%s, %s, %s, %s, { %d, %d, %s, %s }>", e.eventTime, e.retrievalId, e.rootCid, e.providerId, e.receivedBytesSize, e.receivedCidsCount, e.protocol, e.duration) + return fmt.Sprintf("SucceededEvent<%s, %s, %s, %s, { %d, %d, %s, %s }>", e.eventTime, e.retrievalId, e.rootCid, e.endpoint, e.receivedBytesSize, e.receivedCidsCount, e.protocol, e.duration) } func Success(at time.Time, retrievalId types.RetrievalID, candidate types.RetrievalCandidate, receivedBytesSize uint64, receivedCidsCount uint64, duration time.Duration, protocol multicodec.Code) SucceededEvent { - return SucceededEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.MinerPeer.ID}, receivedBytesSize, receivedCidsCount, duration, protocol} + return SucceededEvent{providerRetrievalEvent{retrievalEvent{at, retrievalId, candidate.RootCid}, candidate.Endpoint()}, receivedBytesSize, receivedCidsCount, duration, protocol} } diff --git a/pkg/extractor/carreader.go b/pkg/extractor/carreader.go new file mode 100644 index 00000000..d11c952a --- /dev/null +++ b/pkg/extractor/carreader.go @@ -0,0 +1,136 @@ +package extractor + +import ( + "context" + "fmt" + "io" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" +) + +// ExtractingCarReader reads a CAR stream and extracts UnixFS content to disk, +// tracking expected blocks via a frontier to detect incomplete streams. +type ExtractingCarReader struct { + extractor *Extractor + expected map[cid.Cid]struct{} + onBlock func(int) +} + +func NewExtractingCarReader(ext *Extractor, rootCid cid.Cid) *ExtractingCarReader { + return &ExtractingCarReader{ + extractor: ext, + expected: map[cid.Cid]struct{}{rootCid: {}}, + } +} + +func (r *ExtractingCarReader) OnBlock(cb func(int)) { + r.onBlock = cb +} + +func (r *ExtractingCarReader) ReadAndExtract(ctx context.Context, rdr io.Reader) (uint64, uint64, error) { + br, err := car.NewBlockReader(rdr) + if err != nil { + return 0, 0, fmt.Errorf("failed to create block reader: %w", err) + } + + var blockCount uint64 + var byteCount uint64 + + for { + if ctx.Err() != nil { + return blockCount, byteCount, ctx.Err() + } + + block, err := br.Next() + if err == io.EOF { + break + } + if err != nil { + // stream died mid-transfer — collect remaining expected CIDs + // so the caller can fall back to per-block + missing := make([]cid.Cid, 0, len(r.expected)) + for c := range r.expected { + missing = append(missing, c) + } + return blockCount, byteCount, &IncompleteError{Missing: missing, Cause: err} + } + + c := block.Cid() + data := block.RawData() + + if _, ok := r.expected[c]; !ok { + // duplicate or out-of-order block + if r.extractor.IsProcessed(c) { + continue + } + logger.Debugw("unexpected block in CAR", "cid", c) + } + delete(r.expected, c) + + computed, err := c.Prefix().Sum(data) + if err != nil { + return blockCount, byteCount, fmt.Errorf("failed to compute CID for block: %w", err) + } + if !computed.Equals(c) { + return blockCount, byteCount, fmt.Errorf("CID mismatch: expected %s, got %s", c, computed) + } + + blk, err := blocks.NewBlockWithCid(data, c) + if err != nil { + return blockCount, byteCount, fmt.Errorf("failed to create block: %w", err) + } + + children, err := r.extractor.ProcessBlock(ctx, blk) + if err != nil { + logger.Warnw("extractor failed", "cid", c, "err", err) + } + + for _, child := range children { + if !r.extractor.IsProcessed(child) && !isIdentityCid(child) { + r.expected[child] = struct{}{} + } + } + + blockCount++ + byteCount += uint64(len(data)) + + if r.onBlock != nil { + r.onBlock(len(data)) + } + } + + if len(r.expected) > 0 { + missing := make([]cid.Cid, 0, len(r.expected)) + for c := range r.expected { + missing = append(missing, c) + } + return blockCount, byteCount, &IncompleteError{Missing: missing} + } + + return blockCount, byteCount, nil +} + +type IncompleteError struct { + Missing []cid.Cid + Cause error // non-nil if the stream failed mid-transfer +} + +func (e *IncompleteError) Error() string { + if e.Cause != nil { + return fmt.Sprintf("incomplete CAR: missing %d blocks: %v", len(e.Missing), e.Cause) + } + return fmt.Sprintf("incomplete CAR: missing %d blocks", len(e.Missing)) +} + +func (e *IncompleteError) Unwrap() error { + return e.Cause +} + +func IsMissing(err error) ([]cid.Cid, bool) { + if ie, ok := err.(*IncompleteError); ok { + return ie.Missing, true + } + return nil, false +} diff --git a/pkg/extractor/extractor.go b/pkg/extractor/extractor.go new file mode 100644 index 00000000..34afb4b1 --- /dev/null +++ b/pkg/extractor/extractor.go @@ -0,0 +1,683 @@ +package extractor + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-log/v2" + "github.com/ipfs/go-unixfsnode/data" + dagpb "github.com/ipld/go-codec-dagpb" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/multiformats/go-multihash" +) + +var logger = log.Logger("lassie/extractor") + +func isIdentityCid(c cid.Cid) bool { + return c.Prefix().MhType == multihash.IDENTITY +} + +func extractIdentityData(c cid.Cid) ([]byte, error) { + hash := c.Hash() + decoded, err := multihash.Decode(hash) + if err != nil { + return nil, err + } + return decoded.Digest, nil +} + +type FileProgressCallback func(path string, written, total int64) + +// Extractor writes UnixFS content to disk as blocks arrive, without +// buffering entire files in memory. Handles chunked files, intermediate +// nodes, and out-of-order block arrival. +type Extractor struct { + outputDir string + mu sync.RWMutex + + openFiles map[cid.Cid]*fileWriter + pathContext map[cid.Cid]string + fileRoot map[cid.Cid]cid.Cid // chunk/intermediate CID -> file root CID + processed map[cid.Cid]bool + pendingChunks map[cid.Cid][]byte // blocks that arrived before their parent File node + + onFileStart FileProgressCallback + onFileProgress FileProgressCallback + onFileComplete FileProgressCallback +} + +type chunkPosition struct { + offset int64 + size int64 +} + +type fileWriter struct { + path string + file *os.File + expected int64 + written int64 + positions map[cid.Cid][]chunkPosition + pending int +} + +type ChildLink struct { + Cid cid.Cid + Name string +} + +func New(outputDir string) (*Extractor, error) { + if err := os.MkdirAll(outputDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create output directory: %w", err) + } + return &Extractor{ + outputDir: outputDir, + openFiles: make(map[cid.Cid]*fileWriter), + pathContext: make(map[cid.Cid]string), + fileRoot: make(map[cid.Cid]cid.Cid), + processed: make(map[cid.Cid]bool), + pendingChunks: make(map[cid.Cid][]byte), + }, nil +} + +func (e *Extractor) OnFileStart(cb FileProgressCallback) { e.onFileStart = cb } +func (e *Extractor) OnFileProgress(cb FileProgressCallback) { e.onFileProgress = cb } +func (e *Extractor) OnFileComplete(cb FileProgressCallback) { e.onFileComplete = cb } + +func (e *Extractor) IsProcessed(c cid.Cid) bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.processed[c] +} + +func (e *Extractor) SetRootPath(rootCid cid.Cid, name string) { + sanitized := sanitizeName(name) + if sanitized == "" { + sanitized = rootCid.String() + } + e.mu.Lock() + e.pathContext[rootCid] = sanitized + e.mu.Unlock() +} + +// ProcessBlock decodes a block and returns child CIDs that should be fetched next. +func (e *Extractor) ProcessBlock(ctx context.Context, block blocks.Block) ([]cid.Cid, error) { + c := block.Cid() + + e.mu.RLock() + if e.processed[c] { + e.mu.RUnlock() + logger.Debugw("ProcessBlock: skipping duplicate", "cid", c) + return nil, nil + } + + rootCid, isFileChild := e.fileRoot[c] + e.mu.RUnlock() + + node, err := decodeBlock(block) + if err != nil { + // not DAG-PB — write as chunk if owned, otherwise standalone raw + if isFileChild { + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return nil, e.writeChunk(c, rootCid, block.RawData()) + } + return nil, e.handleRawBlock(c, block.RawData()) + } + + if !node.FieldData().Exists() { + // DAG-PB without UnixFS data + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return e.extractPBLinks(node), nil + } + + ufsData, err := data.DecodeUnixFSData(node.Data.Must().Bytes()) + if err != nil { + if isFileChild { + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return nil, e.writeChunk(c, rootCid, block.RawData()) + } + return nil, e.handleRawBlock(c, block.RawData()) + } + + dataType := ufsData.FieldDataType().Int() + switch dataType { + case data.Data_Directory: + return e.processDirectory(c, node) + case data.Data_File: + if isFileChild { + return e.processIntermediateFile(c, rootCid, node, ufsData) + } + return e.processFile(c, node, ufsData) + case data.Data_Raw: + // UnixFS Raw wraps payload in protobuf — extract inner data, not block bytes + rawData := ufsData.FieldData().Must().Bytes() + if isFileChild { + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return nil, e.writeChunk(c, rootCid, rawData) + } + return nil, e.handleRawBlock(c, rawData) + case data.Data_Symlink: + return nil, e.processSymlink(c, ufsData) + default: + logger.Warnw("unsupported UnixFS type", "cid", c, "type", dataType) + return nil, nil + } +} + +func (e *Extractor) processDirectory(c cid.Cid, node dagpb.PBNode) ([]cid.Cid, error) { + path := e.getPath(c) + if path == "" { + path = c.String() + } + + fullPath := filepath.Join(e.outputDir, path) + if err := os.MkdirAll(fullPath, 0755); err != nil { + return nil, fmt.Errorf("failed to create directory %s: %w", fullPath, err) + } + logger.Debugw("created directory", "path", fullPath) + + var children []cid.Cid + linksIter := node.Links.Iterator() + for !linksIter.Done() { + _, link := linksIter.Next() + linkCid := link.Hash.Link().(cidlink.Link).Cid + name := "" + if link.Name.Exists() { + name = link.Name.Must().String() + } + if name == "" { + name = linkCid.String() + } + childPath, err := e.safePath(path, name) + if err != nil { + logger.Warnw("skipping unsafe path", "cid", linkCid, "name", name, "err", err) + continue + } + e.mu.Lock() + e.pathContext[linkCid] = childPath + e.mu.Unlock() + children = append(children, linkCid) + } + + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return children, nil +} + +func (e *Extractor) processFile(c cid.Cid, node dagpb.PBNode, ufsData data.UnixFSData) ([]cid.Cid, error) { + path := e.getPath(c) + if path == "" { + path = c.String() + } + + fullPath := filepath.Join(e.outputDir, path) + if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { + return nil, fmt.Errorf("failed to create parent directory: %w", err) + } + + linksIter := node.Links.Iterator() + if linksIter.Done() { + // single-block file + var fileData []byte + if ufsData.FieldData().Exists() { + fileData = ufsData.FieldData().Must().Bytes() + } + size := int64(len(fileData)) + if e.onFileStart != nil { + e.onFileStart(path, 0, size) + } + if err := os.WriteFile(fullPath, fileData, 0644); err != nil { + return nil, fmt.Errorf("failed to write file %s: %w", fullPath, err) + } + if e.onFileComplete != nil { + e.onFileComplete(path, size, size) + } + logger.Debugw("wrote single-block file", "path", fullPath, "bytes", len(fileData)) + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return nil, nil + } + + // chunked file + f, err := os.Create(fullPath) + if err != nil { + return nil, fmt.Errorf("failed to create file %s: %w", fullPath, err) + } + + var expectedSize int64 = -1 + if ufsData.FieldFileSize().Exists() { + expectedSize = ufsData.FieldFileSize().Must().Int() + } + + // write inline data before registering in openFiles + inlineOffset := int64(0) + if ufsData.FieldData().Exists() { + inlineData := ufsData.FieldData().Must().Bytes() + if len(inlineData) > 0 { + n, err := f.WriteAt(inlineData, 0) + if err != nil { + f.Close() + return nil, fmt.Errorf("failed to write inline data: %w", err) + } + inlineOffset = int64(n) + } + } + + fw := &fileWriter{ + path: path, + file: f, + expected: expectedSize, + positions: make(map[cid.Cid][]chunkPosition), + } + e.mu.Lock() + e.openFiles[c] = fw + e.mu.Unlock() + + if e.onFileStart != nil { + e.onFileStart(path, 0, expectedSize) + } + + offset := inlineOffset + linksIter = node.Links.Iterator() + idx := 0 + for !linksIter.Done() { + _, link := linksIter.Next() + linkCid := link.Hash.Link().(cidlink.Link).Cid + + var chunkSize int64 + sizes := ufsData.FieldBlockSizes() + if int64(idx) < sizes.Length() { + sizeVal, _ := sizes.LookupByIndex(int64(idx)) + if sizeVal != nil { + chunkSize, _ = sizeVal.AsInt() + } + } + + fw.positions[linkCid] = append(fw.positions[linkCid], chunkPosition{offset: offset, size: chunkSize}) + fw.pending++ + e.mu.Lock() + e.fileRoot[linkCid] = c + e.mu.Unlock() + + offset += chunkSize + idx++ + } + + // deduplicate child CIDs; handle identity CIDs inline + seen := make(map[cid.Cid]bool) + var children []cid.Cid + for linkCid := range fw.positions { + if !seen[linkCid] { + seen[linkCid] = true + + if isIdentityCid(linkCid) { + identData, err := extractIdentityData(linkCid) + if err != nil { + logger.Warnw("failed to extract identity data", "cid", linkCid, "err", err) + } else { + if err := e.writeChunk(linkCid, c, identData); err != nil { + logger.Warnw("failed to write identity chunk", "cid", linkCid, "err", err) + } + } + continue + } + + children = append(children, linkCid) + // drain pending chunks that arrived before this File node + e.mu.Lock() + pendingData, ok := e.pendingChunks[linkCid] + if ok { + delete(e.pendingChunks, linkCid) + } + e.mu.Unlock() + if ok { + if err := e.writeChunk(linkCid, c, pendingData); err != nil { + logger.Warnw("failed to write pending chunk", "cid", linkCid, "err", err) + } + } + } + } + + logger.Debugw("started chunked file", "path", fullPath, "chunks", fw.pending, "uniqueCIDs", len(children), "expectedSize", expectedSize) + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return children, nil +} + +// processIntermediateFile handles File nodes that are children of another File +// (internal nodes in multi-level chunked files). +func (e *Extractor) processIntermediateFile(c, rootCid cid.Cid, node dagpb.PBNode, ufsData data.UnixFSData) ([]cid.Cid, error) { + e.mu.RLock() + fw, ok := e.openFiles[rootCid] + e.mu.RUnlock() + if !ok { + return nil, fmt.Errorf("no open file for intermediate node %s (root %s)", c, rootCid) + } + + myPositions := fw.positions[c] + if len(myPositions) == 0 { + return nil, fmt.Errorf("intermediate node %s has no position in file", c) + } + baseOffset := myPositions[0].offset + + // intermediate nodes don't carry data themselves, just structure + delete(fw.positions, c) + fw.pending -= len(myPositions) + + inlineLen := int64(0) + if ufsData.FieldData().Exists() { + inlineData := ufsData.FieldData().Must().Bytes() + if len(inlineData) > 0 { + _, err := fw.file.WriteAt(inlineData, baseOffset) + if err != nil { + return nil, fmt.Errorf("failed to write intermediate inline data: %w", err) + } + inlineLen = int64(len(inlineData)) + } + } + + offset := baseOffset + inlineLen + linksIter := node.Links.Iterator() + idx := 0 + for !linksIter.Done() { + _, link := linksIter.Next() + linkCid := link.Hash.Link().(cidlink.Link).Cid + + var chunkSize int64 + sizes := ufsData.FieldBlockSizes() + if int64(idx) < sizes.Length() { + sizeVal, _ := sizes.LookupByIndex(int64(idx)) + if sizeVal != nil { + chunkSize, _ = sizeVal.AsInt() + } + } + + fw.positions[linkCid] = append(fw.positions[linkCid], chunkPosition{offset: offset, size: chunkSize}) + fw.pending++ + e.mu.Lock() + e.fileRoot[linkCid] = rootCid + e.mu.Unlock() + + offset += chunkSize + idx++ + } + + seen := make(map[cid.Cid]bool) + var children []cid.Cid + linksIter = node.Links.Iterator() + for !linksIter.Done() { + _, link := linksIter.Next() + linkCid := link.Hash.Link().(cidlink.Link).Cid + if !seen[linkCid] { + seen[linkCid] = true + + if isIdentityCid(linkCid) { + identData, err := extractIdentityData(linkCid) + if err != nil { + logger.Warnw("failed to extract identity data", "cid", linkCid, "err", err) + } else { + if err := e.writeChunk(linkCid, rootCid, identData); err != nil { + logger.Warnw("failed to write identity chunk", "cid", linkCid, "err", err) + } + } + continue + } + + children = append(children, linkCid) + e.mu.Lock() + pendingData, ok := e.pendingChunks[linkCid] + if ok { + delete(e.pendingChunks, linkCid) + } + e.mu.Unlock() + if ok { + if err := e.writeChunk(linkCid, rootCid, pendingData); err != nil { + logger.Warnw("failed to write pending chunk", "cid", linkCid, "err", err) + } + } + } + } + + e.mu.Lock() + delete(e.fileRoot, c) + e.processed[c] = true + e.mu.Unlock() + + logger.Debugw("processed intermediate file node", "cid", c, "root", rootCid, "newChildren", len(children), "baseOffset", baseOffset) + return children, nil +} + +func (e *Extractor) writeChunk(chunkCid, fileCid cid.Cid, rawData []byte) error { + e.mu.RLock() + fw, ok := e.openFiles[fileCid] + e.mu.RUnlock() + if !ok { + return fmt.Errorf("no open file for chunk %s (file %s)", chunkCid, fileCid) + } + + positions := fw.positions[chunkCid] + if len(positions) == 0 { + return fmt.Errorf("chunk %s has no positions in file", chunkCid) + } + + // for DAG-PB chunks, extract the UnixFS data payload + chunkData := rawData + nb := dagpb.Type.PBNode.NewBuilder() + if err := dagpb.DecodeBytes(nb, rawData); err == nil { + pbNode := nb.Build().(dagpb.PBNode) + if pbNode.FieldData().Exists() { + ufsData, err := data.DecodeUnixFSData(pbNode.Data.Must().Bytes()) + if err == nil && ufsData.FieldData().Exists() { + chunkData = ufsData.FieldData().Must().Bytes() + } + } + } + + var bytesWritten int64 + for _, pos := range positions { + _, err := fw.file.WriteAt(chunkData, pos.offset) + if err != nil { + return fmt.Errorf("failed to write chunk at offset %d: %w", pos.offset, err) + } + bytesWritten += int64(len(chunkData)) + fw.pending-- + } + fw.written += bytesWritten + + if e.onFileProgress != nil { + e.onFileProgress(fw.path, fw.written, fw.expected) + } + + delete(fw.positions, chunkCid) + e.mu.Lock() + delete(e.fileRoot, chunkCid) + e.mu.Unlock() + + if fw.pending == 0 { + e.closeFile(fileCid) + } + + return nil +} + +func (e *Extractor) handleRawBlock(c cid.Cid, rawData []byte) error { + e.mu.RLock() + rootCid, ok := e.fileRoot[c] + e.mu.RUnlock() + if ok { + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return e.writeChunk(c, rootCid, rawData) + } + + path := e.getPath(c) + if path != "" { + fullPath := filepath.Join(e.outputDir, path) + if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { + return fmt.Errorf("failed to create parent directory: %w", err) + } + size := int64(len(rawData)) + if e.onFileStart != nil { + e.onFileStart(path, 0, size) + } + if err := os.WriteFile(fullPath, rawData, 0644); err != nil { + return fmt.Errorf("failed to write raw block %s: %w", fullPath, err) + } + if e.onFileComplete != nil { + e.onFileComplete(path, size, size) + } + logger.Debugw("wrote raw block", "path", fullPath, "bytes", len(rawData)) + e.mu.Lock() + e.processed[c] = true + e.mu.Unlock() + return nil + } + + // no path context — block arrived before its parent, stash it + e.mu.Lock() + e.pendingChunks[c] = rawData + e.mu.Unlock() + return nil +} + +func (e *Extractor) processSymlink(c cid.Cid, ufsData data.UnixFSData) error { + logger.Warnw("symlink extraction not implemented", "cid", c) + return nil +} + +func (e *Extractor) extractPBLinks(node dagpb.PBNode) []cid.Cid { + var links []cid.Cid + linksIter := node.Links.Iterator() + for !linksIter.Done() { + _, link := linksIter.Next() + linkCid := link.Hash.Link().(cidlink.Link).Cid + links = append(links, linkCid) + } + return links +} + +func (e *Extractor) getPath(c cid.Cid) string { + e.mu.RLock() + path, ok := e.pathContext[c] + e.mu.RUnlock() + if ok { + return path + } + return "" +} + +func sanitizeName(name string) string { + if name == "" { + return "" + } + name = filepath.Base(name) + if name == "." || name == ".." { + return "" + } + name = strings.TrimSpace(name) + return name +} + +func (e *Extractor) safePath(basePath, name string) (string, error) { + sanitized := sanitizeName(name) + if sanitized == "" { + return "", fmt.Errorf("invalid filename: %q", name) + } + joined := filepath.Join(basePath, sanitized) + fullPath := filepath.Join(e.outputDir, joined) + absOutput, err := filepath.Abs(e.outputDir) + if err != nil { + return "", err + } + absFull, err := filepath.Abs(fullPath) + if err != nil { + return "", err + } + if !strings.HasPrefix(absFull, absOutput+string(filepath.Separator)) && absFull != absOutput { + return "", fmt.Errorf("path traversal attempt: %q", name) + } + return joined, nil +} + +func (e *Extractor) closeFile(c cid.Cid) { + e.mu.Lock() + fw, ok := e.openFiles[c] + if ok { + delete(e.openFiles, c) + } + e.mu.Unlock() + if ok { + fw.file.Close() + if e.onFileComplete != nil { + e.onFileComplete(fw.path, fw.written, fw.expected) + } + logger.Debugw("closed file", "path", fw.path, "expected", fw.expected) + } +} + +func (e *Extractor) Close() error { + e.mu.Lock() + toClose := make(map[cid.Cid]*fileWriter, len(e.openFiles)) + for c, fw := range e.openFiles { + toClose[c] = fw + } + e.openFiles = make(map[cid.Cid]*fileWriter) + pendingCount := len(e.pendingChunks) + e.pendingChunks = make(map[cid.Cid][]byte) + e.mu.Unlock() + + var errs []error + for _, fw := range toClose { + if err := fw.file.Close(); err != nil { + errs = append(errs, fmt.Errorf("closing %s: %w", fw.path, err)) + } + if fw.pending > 0 { + logger.Warnw("incomplete file", "path", fw.path, "pendingChunks", fw.pending, "expected", fw.expected) + } + } + if pendingCount > 0 { + logger.Warnw("orphan pending chunks", "count", pendingCount) + } + if len(errs) > 0 { + return fmt.Errorf("close errors: %v", errs) + } + return nil +} + +func decodeBlock(block blocks.Block) (dagpb.PBNode, error) { + c := block.Cid() + if c.Prefix().Codec != cid.DagProtobuf { + return nil, fmt.Errorf("not a DAG-PB block: codec %x", c.Prefix().Codec) + } + + nb := dagpb.Type.PBNode.NewBuilder() + if err := dagpb.DecodeBytes(nb, block.RawData()); err != nil { + return nil, err + } + node := nb.Build() + pbNode, ok := node.(dagpb.PBNode) + if !ok { + return nil, fmt.Errorf("decoded node is not PBNode: %T", node) + } + return pbNode, nil +} + +var _ io.Closer = (*Extractor)(nil) diff --git a/pkg/extractor/extractor_test.go b/pkg/extractor/extractor_test.go new file mode 100644 index 00000000..355bc515 --- /dev/null +++ b/pkg/extractor/extractor_test.go @@ -0,0 +1,337 @@ +package extractor + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode/data/builder" + dagpb "github.com/ipld/go-codec-dagpb" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/storage/memstore" +) + +func buildWithLinkSystem(t *testing.T, content []byte) (blocks.Block, *memstore.Store) { + t.Helper() + + store := &memstore.Store{} + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(store) + lsys.SetReadStorage(store) + + link, _, err := builder.BuildUnixFSFile(bytes.NewReader(content), "", &lsys) + if err != nil { + t.Fatalf("BuildUnixFSFile failed: %v", err) + } + + cidLink := link.(cidlink.Link) + data, err := store.Get(context.Background(), cidLink.Cid.KeyString()) + if err != nil { + t.Fatalf("failed to get block from store: %v", err) + } + + block, _ := blocks.NewBlockWithCid(data, cidLink.Cid) + return block, store +} + +func TestExtractSingleBlockFile(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + ext, err := New(tmpDir) + if err != nil { + t.Fatalf("failed to create extractor: %v", err) + } + defer ext.Close() + + fileContent := []byte("hello world") + block, _ := buildWithLinkSystem(t, fileContent) + + ext.SetRootPath(block.Cid(), "test.txt") + + children, err := ext.ProcessBlock(ctx, block) + if err != nil { + t.Fatalf("ProcessBlock failed: %v", err) + } + if len(children) != 0 { + t.Errorf("expected 0 children, got %d", len(children)) + } + + content, err := os.ReadFile(filepath.Join(tmpDir, "test.txt")) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + if string(content) != "hello world" { + t.Errorf("content mismatch: got %q, want %q", content, "hello world") + } +} + +func TestExtractDirectory(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + ext, err := New(tmpDir) + if err != nil { + t.Fatalf("failed to create extractor: %v", err) + } + defer ext.Close() + + store := &memstore.Store{} + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(store) + lsys.SetReadStorage(store) + + fileContent := []byte("file content") + fileLink, fileSize, err := builder.BuildUnixFSFile(bytes.NewReader(fileContent), "", &lsys) + if err != nil { + t.Fatalf("BuildUnixFSFile failed: %v", err) + } + fileCidLink := fileLink.(cidlink.Link) + + entry, err := builder.BuildUnixFSDirectoryEntry("myfile.txt", int64(fileSize), fileLink) + if err != nil { + t.Fatalf("BuildUnixFSDirectoryEntry failed: %v", err) + } + + dirLink, _, err := builder.BuildUnixFSDirectory([]dagpb.PBLink{entry}, &lsys) + if err != nil { + t.Fatalf("BuildUnixFSDirectory failed: %v", err) + } + dirCidLink := dirLink.(cidlink.Link) + + dirData, _ := store.Get(ctx, dirCidLink.Cid.KeyString()) + dirBlock, _ := blocks.NewBlockWithCid(dirData, dirCidLink.Cid) + + fileData, _ := store.Get(ctx, fileCidLink.Cid.KeyString()) + fileBlock, _ := blocks.NewBlockWithCid(fileData, fileCidLink.Cid) + + ext.SetRootPath(dirCidLink.Cid, "testdir") + + children, err := ext.ProcessBlock(ctx, dirBlock) + if err != nil { + t.Fatalf("ProcessBlock (dir) failed: %v", err) + } + if len(children) != 1 { + t.Fatalf("expected 1 child, got %d", len(children)) + } + + info, err := os.Stat(filepath.Join(tmpDir, "testdir")) + if err != nil { + t.Fatalf("directory not created: %v", err) + } + if !info.IsDir() { + t.Error("expected directory") + } + + _, err = ext.ProcessBlock(ctx, fileBlock) + if err != nil { + t.Fatalf("ProcessBlock (file) failed: %v", err) + } + + content, err := os.ReadFile(filepath.Join(tmpDir, "testdir", "myfile.txt")) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + if string(content) != "file content" { + t.Errorf("content mismatch: got %q", content) + } +} + +func TestSanitizeName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"normal.txt", "normal.txt"}, + {"../escape", "escape"}, + {"../../etc/passwd", "passwd"}, + {"/absolute/path", "path"}, + {".", ""}, + {"..", ""}, + {" spaces ", "spaces"}, + {"", ""}, + } + + for _, tc := range tests { + got := sanitizeName(tc.input) + if got != tc.expected { + t.Errorf("sanitizeName(%q) = %q, want %q", tc.input, got, tc.expected) + } + } +} + +func TestSetRootPathSanitization(t *testing.T) { + tmpDir := t.TempDir() + ext, _ := New(tmpDir) + defer ext.Close() + + maliciousCid := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") + ext.SetRootPath(maliciousCid, "../../etc/passwd") + + path := ext.getPath(maliciousCid) + if path == "../../etc/passwd" { + t.Error("path traversal not sanitized") + } + if path != "passwd" { + t.Errorf("expected 'passwd', got %q", path) + } +} + +func TestExtractChunkedFile(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + ext, err := New(tmpDir) + if err != nil { + t.Fatalf("failed to create extractor: %v", err) + } + defer ext.Close() + + // varied content so each 1024-byte chunk gets a unique CID + fileContent := make([]byte, 3000) + for i := range fileContent { + // mix in i/1024 to make each chunk unique + fileContent[i] = byte(i + i/1024*100) + } + + store := &memstore.Store{} + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(store) + lsys.SetReadStorage(store) + + link, _, err := builder.BuildUnixFSFile(bytes.NewReader(fileContent), "size-1024", &lsys) + if err != nil { + t.Fatalf("BuildUnixFSFile failed: %v", err) + } + rootCidLink := link.(cidlink.Link) + + ext.SetRootPath(rootCidLink.Cid, "chunked.txt") + + rootData, _ := store.Get(ctx, rootCidLink.Cid.KeyString()) + rootBlock, _ := blocks.NewBlockWithCid(rootData, rootCidLink.Cid) + + children, err := ext.ProcessBlock(ctx, rootBlock) + if err != nil { + t.Fatalf("ProcessBlock (root) failed: %v", err) + } + if len(children) == 0 { + t.Fatal("expected children for chunked file") + } + t.Logf("root block has %d children", len(children)) + + // DFS traversal of child blocks + queue := children + for len(queue) > 0 { + childCid := queue[0] + queue = queue[1:] + + childData, err := store.Get(ctx, childCid.KeyString()) + if err != nil { + t.Fatalf("failed to get child block %s: %v", childCid, err) + } + childBlock, _ := blocks.NewBlockWithCid(childData, childCid) + grandchildren, err := ext.ProcessBlock(ctx, childBlock) + if err != nil { + t.Fatalf("ProcessBlock (chunk %s) failed: %v", childCid, err) + } + queue = append(queue, grandchildren...) + } + + content, err := os.ReadFile(filepath.Join(tmpDir, "chunked.txt")) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + if len(content) != len(fileContent) { + t.Errorf("content length mismatch: got %d, want %d", len(content), len(fileContent)) + } + if !bytes.Equal(content, fileContent) { + t.Error("content mismatch") + } +} + +func TestDuplicateBlockHandling(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + ext, _ := New(tmpDir) + defer ext.Close() + + fileContent := []byte("test content") + block, _ := buildWithLinkSystem(t, fileContent) + + ext.SetRootPath(block.Cid(), "dup.txt") + + ext.ProcessBlock(ctx, block) + ext.ProcessBlock(ctx, block) // should be skipped + info, _ := os.Stat(filepath.Join(tmpDir, "dup.txt")) + if info.Size() != int64(len(fileContent)) { + t.Errorf("file size %d, expected %d (duplicate not handled)", info.Size(), len(fileContent)) + } +} + +func TestDuplicateChunksInFile(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + ext, err := New(tmpDir) + if err != nil { + t.Fatalf("failed to create extractor: %v", err) + } + defer ext.Close() + + // 3 identical 1024-byte chunks → same CID repeated in links + chunk := bytes.Repeat([]byte("x"), 1024) + fileContent := bytes.Repeat(chunk, 3) // 3072 bytes, 3 identical 1024-byte chunks + + store := &memstore.Store{} + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(store) + lsys.SetReadStorage(store) + + link, _, err := builder.BuildUnixFSFile(bytes.NewReader(fileContent), "size-1024", &lsys) + if err != nil { + t.Fatalf("BuildUnixFSFile failed: %v", err) + } + rootCidLink := link.(cidlink.Link) + + ext.SetRootPath(rootCidLink.Cid, "repeated.txt") + + rootData, _ := store.Get(ctx, rootCidLink.Cid.KeyString()) + rootBlock, _ := blocks.NewBlockWithCid(rootData, rootCidLink.Cid) + + children, err := ext.ProcessBlock(ctx, rootBlock) + if err != nil { + t.Fatalf("ProcessBlock (root) failed: %v", err) + } + + // deduplicated: 1 unique CID for 3 chunk positions + t.Logf("root returned %d unique children for file with repeated chunks", len(children)) + + for _, childCid := range children { + childData, err := store.Get(ctx, childCid.KeyString()) + if err != nil { + t.Fatalf("failed to get child block %s: %v", childCid, err) + } + childBlock, _ := blocks.NewBlockWithCid(childData, childCid) + _, err = ext.ProcessBlock(ctx, childBlock) + if err != nil { + t.Fatalf("ProcessBlock (chunk %s) failed: %v", childCid, err) + } + } + + // all 3072 bytes, not just 1024 — repeated chunks must be expanded + content, err := os.ReadFile(filepath.Join(tmpDir, "repeated.txt")) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + if len(content) != len(fileContent) { + t.Errorf("content length mismatch: got %d, want %d (duplicate chunks not expanded)", len(content), len(fileContent)) + } + if !bytes.Equal(content, fileContent) { + t.Error("content mismatch") + } +} diff --git a/pkg/heyfil/heyfil.go b/pkg/heyfil/heyfil.go deleted file mode 100644 index 8e30906d..00000000 --- a/pkg/heyfil/heyfil.go +++ /dev/null @@ -1,173 +0,0 @@ -package heyfil - -import ( - "encoding/json" - "fmt" - "net/http" - "strings" - "sync" - - "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p/core/peer" - "go.uber.org/multierr" -) - -const DefaultHeyfilEndpoint = "https://heyfil.prod.cid.contact/" - -var logger = log.Logger("lassie/heyfil") - -func isPeerId(s string) bool { - _, err := peer.Decode(s) - return err == nil -} - -func isFilecoinFaddr(s string) bool { - // quick check if matches /^f0\d+$/ { - if len(s) > 2 && s[0] == 'f' && s[1] == '0' { - for _, c := range s[2:] { - if c < '0' || c > '9' { - return false - } - } - return true - } - return false -} - -type Heyfil struct { - Endpoint string - TranslatePeerId bool - TranslateFaddr bool -} - -func (h Heyfil) CanTranslate(s string) bool { - return !strings.Contains(s, "/") && ((h.TranslatePeerId && isPeerId(s)) || (h.TranslateFaddr && isFilecoinFaddr(s))) -} - -func (h Heyfil) endpoint() string { - if h.Endpoint == "" { - return DefaultHeyfilEndpoint - } - return h.Endpoint -} - -// TranslateAll performs a Translate on all strings in the input slice. If none -// of the strings can be translated, the input slice is returned as-is. -func (h Heyfil) TranslateAll(ss []string) ([]string, error) { - translated := make([]string, len(ss)) - var merr error - var translatedLk sync.Mutex - var wg sync.WaitGroup - for ii, s := range ss { - if !h.CanTranslate(s) { - translated[ii] = s - continue - } - wg.Add(1) - go func(ii int, s string) { - defer wg.Done() - res, err := h.Translate(s) - if err != nil { - translatedLk.Lock() - multierr.AppendInto(&merr, err) - translatedLk.Unlock() - return - } - translatedLk.Lock() - translated[ii] = res - translatedLk.Unlock() - }(ii, s) - } - wg.Wait() - if merr != nil { - return nil, merr - } - return translated, nil -} - -// Translate will translate an input string to a full multiaddr if the string -// appears to be a Filecoin SP actor address or a peer id using the Heyfil -// service. If the input string is not a Filecoin SP actor address or a peer id, -// it will be returned as-is. -func (h Heyfil) Translate(s string) (string, error) { - if h.TranslatePeerId && isPeerId(s) { - res, err := h.translatePeerId(s) - if err != nil { - logger.Debugw("failed to translate peer id", "input", s, "err", err) - return "", err - } - return res, nil - } - if h.TranslateFaddr && isFilecoinFaddr(s) { - res, err := h.translateFaddr(s) - if err != nil { - logger.Debugw("failed to translate faddr", "input", s, "err", err) - return "", err - } - return res, nil - } - return s, nil // pass-through -} - -func (h Heyfil) translatePeerId(s string) (string, error) { - url := h.endpoint() + "/sp?peerid=" + s - logger.Debugw("translating peer id", "input", s, "url", url) - got := make([]string, 0) - if err := httpGet(url, &got); err != nil { - return "", err - } - if len(got) == 0 { - return "", fmt.Errorf("expected at least 1 response, got %d", len(got)) - } - return h.translateFaddr(got[0]) -} - -func (h Heyfil) translateFaddr(s string) (string, error) { - url := h.endpoint() + "/sp/" + s - got := make(map[string]interface{}) - if err := httpGet(url, &got); err != nil { - return "", err - } - addrInfo, ok := got["addr_info"].(map[string]interface{}) - if !ok { - return "", fmt.Errorf("expected addr_info to be map[string]interface{}, got %T", got["addr_info"]) - } - id, ok := addrInfo["ID"].(string) - if !ok || id == "" { - return "", fmt.Errorf("expected addr_info.ID to be string") - } - addrs, ok := addrInfo["Addrs"].([]interface{}) - if !ok { - return "", fmt.Errorf("expected addr_info.Addrs to be []interface{}, got %T", addrInfo["Addrs"]) - } - if len(addrs) == 0 { - return "", fmt.Errorf("expected addr_info.Addrs to be non-empty") - } - // TODO: only using the first multiaddr for now, this could be expanded to use - // all of them, either as separate addrs in the translated string, or, we could - // allow a translate that spits out a full AddrInfo that maps directly to the - // value here. TranslateAll([]string) ([]string, error) could become - // TranslateAll([]string) ([]AddrInfo, []string, error)? - addr, ok := addrs[0].(string) - if !ok { - return "", fmt.Errorf("expected addr_info.Addrs[0] to be string, got %T", addrs[0]) - } - return addr + "/p2p/" + id, nil -} - -func httpGet[T any](url string, result *T) error { - logger.Debugf("http get: %s", url) - resp, err := http.Get(url) - if err != nil { - return fmt.Errorf("http get: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("http get: %s", resp.Status) - } - err = json.NewDecoder(resp.Body).Decode(result) - if err != nil { - return fmt.Errorf("decode: %w", err) - } - return nil -} diff --git a/pkg/heyfil/heyfil_test.go b/pkg/heyfil/heyfil_test.go deleted file mode 100644 index 73442a53..00000000 --- a/pkg/heyfil/heyfil_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package heyfil_test - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/filecoin-project/lassie/pkg/heyfil" - "github.com/stretchr/testify/require" -) - -func TestCanHeyfil(t *testing.T) { - testCase := []struct { - name string - input string - canWithPeerID bool - canWithFaddr bool - }{ - {"empty", "", false, false}, - {"faddr", "f01234", false, true}, - {"faddr non miner", "f1234", false, false}, - {"bad faddr", "f00cafebeef", false, false}, - {"p2p", "12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", true, false}, - {"cid p2p", "QmUA9D3H7HeCYsirB3KmPSvZh3dNXMZas6Lwgr4fv1HTTp", true, false}, - {"cidv1 p2p", "bafzbeicwot2npbkuyjppqaoibbohqemn5dnbidt66mjdfso725vm4lmmlm", true, false}, - {"cidv1 not p2p", "bafybeicwot2npbkuyjppqaoibbohqemn5dnbidt66mjdfso725vm4lmmlm", false, false}, - {"multiaddr", "/dns4/dag.w3s.link/tcp/443/https", false, false}, - {"http addr", "http://dag.w3s.link:443", false, false}, - {"multiaddr long", "/dns4/dag.w3s.link/tcp/443/https/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", false, false}, - {"multiaddr ip4", "/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA", false, false}, - } - - for _, tc := range testCase { - t.Run(tc.name, func(t *testing.T) { - require.False(t, heyfil.Heyfil{}.CanTranslate(tc.input)) - require.Equal(t, heyfil.Heyfil{TranslatePeerId: true}.CanTranslate(tc.input), tc.canWithPeerID) - require.Equal(t, heyfil.Heyfil{TranslateFaddr: true}.CanTranslate(tc.input), tc.canWithFaddr) - }) - } -} - -func TestHeyfil(t *testing.T) { - ts := newHeyfilServer() - defer ts.Close() - - trans, err := heyfil.Heyfil{Endpoint: ts.URL}.Translate("12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ") - require.NoError(t, err) - require.Equal(t, "12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ", trans) - trans, err = heyfil.Heyfil{Endpoint: ts.URL, TranslatePeerId: true}.Translate("12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ") - require.NoError(t, err) - require.Equal(t, "/ip4/85.11.148.122/tcp/24001/p2p/12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ", trans) - - trans, err = heyfil.Heyfil{Endpoint: ts.URL}.Translate("f0127896") - require.NoError(t, err) - require.Equal(t, "f0127896", trans) - trans, err = heyfil.Heyfil{Endpoint: ts.URL, TranslateFaddr: true}.Translate("f0127896") - require.NoError(t, err) - require.Equal(t, "/ip4/85.11.148.122/tcp/24001/p2p/12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ", trans) - - // no translation, pass-through - for _, inp := range []string{ - "/dns4/dag.w3s.link/tcp/443/https", - "http://dag.w3s.link:443", - "/dns4/dag.w3s.link/tcp/443/https/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", - "/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA", - "WOT?", // this is an error for another layer ... - } { - trans, err := heyfil.Heyfil{Endpoint: ts.URL, TranslatePeerId: true, TranslateFaddr: true}.Translate(inp) - require.NoError(t, err) - require.Equal(t, trans, inp) - } -} - -func TestHeyfilTranslateAll(t *testing.T) { - ts := newHeyfilServer() - defer ts.Close() - - testData := []struct { - addr string - want string - ispeerid bool - isfaddr bool - }{ - {"/dns4/dag.w3s.link/tcp/443/https", "/dns4/dag.w3s.link/tcp/443/https", false, false}, - {"http://dag.w3s.link:443", "http://dag.w3s.link:443", false, false}, - {"12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ", "/ip4/85.11.148.122/tcp/24001/p2p/12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ", true, false}, - {"/dns4/dag.w3s.link/tcp/443/https/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", "/dns4/dag.w3s.link/tcp/443/https/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", false, false}, - {"/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA", "/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA", false, false}, - {"f0127896", "/ip4/85.11.148.122/tcp/24001/p2p/12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ", false, true}, - {"WOT?", "WOT?", false, false}, // this is an error for another layer ..."WOT?" - } - input := make([]string, len(testData)) - for ii, td := range testData { - input[ii] = td.addr - } - - trans, err := heyfil.Heyfil{Endpoint: ts.URL}.TranslateAll(input) - require.NoError(t, err) - for ii, td := range testData { - want := td.want - if td.isfaddr || td.ispeerid { - want = td.addr - } - require.Equal(t, trans[ii], want) - } - - trans, err = heyfil.Heyfil{Endpoint: ts.URL, TranslatePeerId: true}.TranslateAll(input) - require.NoError(t, err) - for ii, td := range testData { - want := td.want - if td.isfaddr { - want = td.addr - } - require.Equal(t, trans[ii], want) - } - - trans, err = heyfil.Heyfil{Endpoint: ts.URL, TranslateFaddr: true}.TranslateAll(input) - require.NoError(t, err) - for ii, td := range testData { - want := td.want - if td.ispeerid { - want = td.addr - } - require.Equal(t, trans[ii], want) - } - - trans, err = heyfil.Heyfil{Endpoint: ts.URL, TranslateFaddr: true, TranslatePeerId: true}.TranslateAll(input) - require.NoError(t, err) - for ii, td := range testData { - require.Equal(t, trans[ii], td.want) - } -} - -func newHeyfilServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/sp/f0127896" { - w.Write([]byte(`{"id":"f0127896","status":6,"addr_info":{"ID":"12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ","Addrs":["/ip4/85.11.148.122/tcp/24001"]},"last_checked":"2023-09-29T05:57:19.193226313Z","err":"failed to dial 12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ:\n * [/ip4/85.11.148.122/tcp/24001] dial tcp4 85.11.148.122:24001: connect: connection refused","head":null,"known_by_indexer":true,"state_miner_power":{"HasMinPower":false,"MinerPower":{"QualityAdjPower":"0","RawBytePower":"0"},"TotalPower":{"QualityAdjPower":"27973870915671523328","RawBytePower":"12090904615566966784"}},"deal_count":2031}`)) - return - } - if r.URL.Path == "/sp" && r.URL.RawQuery == "peerid=12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ" { - w.Write([]byte(`["f0127896"]`)) - return - } - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("NO JSON FOR YOU!")) - })) -} diff --git a/pkg/indexerlookup/candidatesource.go b/pkg/indexerlookup/candidatesource.go index b47679f2..87b36ee7 100644 --- a/pkg/indexerlookup/candidatesource.go +++ b/pkg/indexerlookup/candidatesource.go @@ -1,20 +1,17 @@ package indexerlookup import ( - "bufio" "context" "encoding/json" - "errors" "fmt" "io" "net/http" + "strings" "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-log/v2" - "github.com/ipni/go-libipni/find/model" - "github.com/ipni/go-libipni/metadata" - "github.com/multiformats/go-multihash" + "github.com/multiformats/go-multibase" ) var ( @@ -37,45 +34,36 @@ func NewCandidateSource(o ...Option) (*IndexerCandidateSource, error) { }, nil } -func decodeMetadata(pr model.ProviderResult) (metadata.Metadata, error) { - if len(pr.Metadata) == 0 { - return metadata.Metadata{}, errors.New("no metadata") - } - // Metadata may contain more than one protocol, sorted by ascending order of their protocol ID. - // Therefore, decode the metadata as metadata.Metadata, then check if it supports Graphsync. - // See: https://github.com/ipni/specs/blob/main/IPNI.md#metadata - dtm := metadata.Default.New() - if err := dtm.UnmarshalBinary(pr.Metadata); err != nil { - logger.Debugw("Failed to unmarshal metadata", "err", err) - return metadata.Metadata{}, err - } - return dtm, nil -} - func (idxf *IndexerCandidateSource) FindCandidates(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error { req, err := idxf.newFindHttpRequest(ctx, c) if err != nil { return err } - req.Header.Set("Accept", "application/x-ndjson") + req.Header.Set("Accept", "application/json") logger.Debugw("sending outgoing request", "url", req.URL, "accept", req.Header.Get("Accept")) resp, err := idxf.httpClient.Do(req) if err != nil { - logger.Debugw("Failed to perform streaming lookup", "err", err) + logger.Debugw("Failed to perform lookup", "err", err) return err } + defer resp.Body.Close() + switch resp.StatusCode { case http.StatusOK: - return idxf.decodeProviderResultStream(ctx, c, resp.Body, cb) + return idxf.decodeDelegatedRoutingResponse(ctx, c, resp.Body, cb) case http.StatusNotFound: return nil + case http.StatusTooManyRequests: + retryAfter := resp.Header.Get("Retry-After") + logger.Debugw("Rate limited by delegated routing server", "retry-after", retryAfter) + return fmt.Errorf("rate limited (429), retry after: %s", retryAfter) default: - return fmt.Errorf("batch find query failed: %v", http.StatusText(resp.StatusCode)) + return fmt.Errorf("provider lookup failed: %v", http.StatusText(resp.StatusCode)) } } func (idxf *IndexerCandidateSource) newFindHttpRequest(ctx context.Context, c cid.Cid) (*http.Request, error) { - endpoint := idxf.findByMultihashEndpoint(c.Hash()) + endpoint := idxf.findByDelegatedRoutingEndpoint(c) req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, err @@ -83,56 +71,103 @@ func (idxf *IndexerCandidateSource) newFindHttpRequest(ctx context.Context, c ci if idxf.httpUserAgent != "" { req.Header.Set("User-Agent", idxf.httpUserAgent) } - if idxf.ipfsDhtCascade { - query := req.URL.Query() - query.Add("cascade", "ipfs-dht") - req.URL.RawQuery = query.Encode() - } - if idxf.legacyCascade { + + // Add filter-protocols query parameter if protocols are specified + // Per https://specs.ipfs.tech/routing/http-routing-v1/ + // Note: cid.contact doesn't support this yet, so we also do client-side filtering + if len(idxf.protocols) > 0 { query := req.URL.Query() - query.Add("cascade", "legacy") + protocols := make([]string, len(idxf.protocols)) + for i, p := range idxf.protocols { + protocols[i] = p.String() + } + query.Add("filter-protocols", strings.Join(protocols, ",")) req.URL.RawQuery = query.Encode() } + return req, nil } -func (idxf *IndexerCandidateSource) decodeProviderResultStream(ctx context.Context, c cid.Cid, from io.ReadCloser, cb func(types.RetrievalCandidate)) error { - defer from.Close() - scanner := bufio.NewScanner(from) - for { +func (idxf *IndexerCandidateSource) decodeDelegatedRoutingResponse(ctx context.Context, c cid.Cid, from io.ReadCloser, cb func(types.RetrievalCandidate)) error { + // Read the entire response body + body, err := io.ReadAll(from) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + // Parse as delegated routing response + var response DelegatedRoutingResponse + if err := json.Unmarshal(body, &response); err != nil { + return fmt.Errorf("failed to unmarshal delegated routing response: %w", err) + } + + // Process each provider + for _, provider := range response.Providers { select { case <-ctx.Done(): return ctx.Err() default: - if scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - var pr model.ProviderResult - if err := json.Unmarshal(line, &pr); err != nil { - return err - } - // skip results without decodable metadata - if md, err := decodeMetadata(pr); err == nil { - var candidate types.RetrievalCandidate - if pr.Provider != nil { - candidate.MinerPeer = *pr.Provider - } - candidate.RootCid = c - candidate.Metadata = md - cb(candidate) - } - } else if err := scanner.Err(); err != nil { - return err - } else { - // There are no more lines remaining to scan as we have reached EOF. - return nil + // Convert to AddrInfo + addrInfo, err := provider.ToAddrInfo() + if err != nil { + logger.Debugw("Failed to convert provider to AddrInfo, skipping", "id", provider.ID, "err", err) + continue } + + // Client-side protocol filtering (fallback for non-compliant servers) + // If protocols are specified, check if provider has any matching protocol + if len(idxf.protocols) > 0 && !idxf.providerMatchesProtocols(&provider) { + logger.Debugw("Provider protocols don't match filter, skipping", "id", provider.ID, "protocols", provider.Protocols, "filter", idxf.protocols) + continue + } + + // Convert protocols to metadata + md, err := provider.ToMetadata() + if err != nil { + logger.Debugw("Failed to convert provider metadata, skipping", "id", provider.ID, "err", err) + continue + } + + // Create candidate and callback + candidate := types.RetrievalCandidate{ + MinerPeer: *addrInfo, + RootCid: c, + Metadata: md, + } + cb(candidate) + } + } + + return nil +} + +func (idxf *IndexerCandidateSource) providerMatchesProtocols(provider *DelegatedProvider) bool { + allowed := make(map[string]bool) + for _, p := range idxf.protocols { + allowed[p.String()] = true + } + + for _, protoName := range provider.Protocols { + if allowed[protoName] { + return true } } + return false } -func (idxf *IndexerCandidateSource) findByMultihashEndpoint(mh multihash.Multihash) string { - return idxf.httpEndpoint.JoinPath("multihash", mh.B58String()).String() +// findByDelegatedRoutingEndpoint constructs the delegated routing API endpoint for a CID +// Normalizes CID to v1 base32 for better HTTP caching as per the spec +func (idxf *IndexerCandidateSource) findByDelegatedRoutingEndpoint(c cid.Cid) string { + // Convert to CIDv1 for better HTTP caching + cidV1 := cid.NewCidV1(c.Type(), c.Hash()) + + // Encode as base32 (the recommended format for HTTP) + cidStr, err := cidV1.StringOfBase(multibase.Base32) + if err != nil { + // Fallback to default string representation if base32 encoding fails + cidStr = cidV1.String() + logger.Debugw("Failed to encode CID as base32, using default", "err", err) + } + + return idxf.httpEndpoint.JoinPath("routing", "v1", "providers", cidStr).String() } diff --git a/pkg/indexerlookup/candidatesource_test.go b/pkg/indexerlookup/candidatesource_test.go index ce275898..60f35ef3 100644 --- a/pkg/indexerlookup/candidatesource_test.go +++ b/pkg/indexerlookup/candidatesource_test.go @@ -11,15 +11,15 @@ import ( "github.com/filecoin-project/lassie/pkg/internal/mockindexer" "github.com/filecoin-project/lassie/pkg/internal/testutil" "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipfs/go-cid" + goipfscid "github.com/ipfs/go-cid" "github.com/ipni/go-libipni/find/model" "github.com/stretchr/testify/require" ) func TestCandidateSource(t *testing.T) { - cids := make([]cid.Cid, 0, 10) - candidates := make(map[cid.Cid][]types.RetrievalCandidate, 10) - binaryMetadata := make(map[cid.Cid][][]byte, 10) + cids := make([]goipfscid.Cid, 0, 10) + candidates := make(map[goipfscid.Cid][]types.RetrievalCandidate, 10) + binaryMetadata := make(map[goipfscid.Cid][][]byte, 10) for i := 0; i < 10; i++ { next := testutil.GenerateRetrievalCandidates(t, 2) cids = append(cids, next[0].RootCid) @@ -32,13 +32,13 @@ func TestCandidateSource(t *testing.T) { } testCases := []struct { name string - cidReturns map[cid.Cid][]model.ProviderResult - expectedReturns map[cid.Cid][]types.RetrievalCandidate + cidReturns map[goipfscid.Cid][]model.ProviderResult + expectedReturns map[goipfscid.Cid][]types.RetrievalCandidate async bool }{ { name: "basic fetch", - cidReturns: map[cid.Cid][]model.ProviderResult{ + cidReturns: map[goipfscid.Cid][]model.ProviderResult{ cids[0]: { { Metadata: binaryMetadata[cids[0]][0], @@ -64,15 +64,15 @@ func TestCandidateSource(t *testing.T) { }, }, }, - expectedReturns: map[cid.Cid][]types.RetrievalCandidate{ + expectedReturns: map[goipfscid.Cid][]types.RetrievalCandidate{ cids[0]: candidates[cids[0]], cids[1]: candidates[cids[1]], }, }, { name: "not found", - cidReturns: map[cid.Cid][]model.ProviderResult{}, - expectedReturns: map[cid.Cid][]types.RetrievalCandidate{ + cidReturns: map[goipfscid.Cid][]model.ProviderResult{}, + expectedReturns: map[goipfscid.Cid][]types.RetrievalCandidate{ cids[0]: {}, cids[1]: {}, }, @@ -109,7 +109,11 @@ func TestCandidateSource(t *testing.T) { case <-ctx.Done(): req.FailNow("cancelled") case recv := <-connectCh: - req.Regexp("^/multihash/"+cid.Hash().B58String(), recv) + // Expect delegated routing endpoint with CIDv1 base32 + cidV1 := goipfscid.NewCidV1(cid.Type(), cid.Hash()) + req.Regexp("^/routing/v1/providers/", recv) + // The CID will be in base32 format + req.Contains(recv, cidV1.String()) } for range expectedReturns { clock.Add(5 * time.Second) diff --git a/pkg/indexerlookup/delegated.go b/pkg/indexerlookup/delegated.go new file mode 100644 index 00000000..ac080ce3 --- /dev/null +++ b/pkg/indexerlookup/delegated.go @@ -0,0 +1,113 @@ +package indexerlookup + +import ( + "encoding/json" + "fmt" + + "github.com/ipni/go-libipni/metadata" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" +) + +// DelegatedRoutingResponse represents the HTTP Routing V1 API response format +// See: https://specs.ipfs.tech/routing/http-routing-v1/#get-routing-v1-providers-cid +type DelegatedRoutingResponse struct { + Providers []DelegatedProvider `json:"Providers"` +} + +// DelegatedProvider represents a single provider in the delegated routing response +type DelegatedProvider struct { + Schema string `json:"Schema"` + ID string `json:"ID"` + Addrs []string `json:"Addrs,omitempty"` + Protocols []string `json:"Protocols,omitempty"` + Metadata map[string]interface{} `json:"-"` // Capture all additional fields +} + +// UnmarshalJSON implements custom JSON unmarshaling to capture protocol-specific metadata +func (dp *DelegatedProvider) UnmarshalJSON(data []byte) error { + // First unmarshal into a map to capture all fields + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + // Unmarshal known fields + if schemaRaw, ok := raw["Schema"]; ok { + json.Unmarshal(schemaRaw, &dp.Schema) + } + if idRaw, ok := raw["ID"]; ok { + json.Unmarshal(idRaw, &dp.ID) + } + if addrsRaw, ok := raw["Addrs"]; ok { + json.Unmarshal(addrsRaw, &dp.Addrs) + } + if protocolsRaw, ok := raw["Protocols"]; ok { + json.Unmarshal(protocolsRaw, &dp.Protocols) + } + + // Capture protocol-specific metadata (any field that's not a known field) + dp.Metadata = make(map[string]interface{}) + knownFields := map[string]bool{ + "Schema": true, "ID": true, "Addrs": true, "Protocols": true, + } + for key, val := range raw { + if !knownFields[key] { + var v interface{} + json.Unmarshal(val, &v) + dp.Metadata[key] = v + } + } + + return nil +} + +// ToAddrInfo converts the delegated provider to a libp2p peer.AddrInfo +func (dp *DelegatedProvider) ToAddrInfo() (*peer.AddrInfo, error) { + peerID, err := peer.Decode(dp.ID) + if err != nil { + return nil, fmt.Errorf("failed to decode peer ID %s: %w", dp.ID, err) + } + + var addrs []multiaddr.Multiaddr + for _, addrStr := range dp.Addrs { + addr, err := multiaddr.NewMultiaddr(addrStr) + if err != nil { + logger.Debugw("Failed to parse multiaddr, skipping", "addr", addrStr, "err", err) + continue + } + addrs = append(addrs, addr) + } + + return &peer.AddrInfo{ + ID: peerID, + Addrs: addrs, + }, nil +} + +// ToMetadata converts the delegated provider's protocol information to metadata.Metadata +func (dp *DelegatedProvider) ToMetadata() (metadata.Metadata, error) { + if len(dp.Protocols) == 0 { + return metadata.Metadata{}, fmt.Errorf("no protocols specified") + } + + var protocols []metadata.Protocol + + for _, protoName := range dp.Protocols { + switch protoName { + case "transport-ipfs-gateway-http": + // HTTP gateway protocol has no additional metadata beyond the protocol ID + protocols = append(protocols, metadata.IpfsGatewayHttp{}) + + default: + logger.Debugw("Unknown protocol, skipping", "protocol", protoName) + continue + } + } + + if len(protocols) == 0 { + return metadata.Metadata{}, fmt.Errorf("no supported protocols found") + } + + return metadata.Default.New(protocols...), nil +} diff --git a/pkg/indexerlookup/options.go b/pkg/indexerlookup/options.go index 8f72c98e..75fa4a51 100644 --- a/pkg/indexerlookup/options.go +++ b/pkg/indexerlookup/options.go @@ -4,6 +4,8 @@ import ( "net/http" "net/url" "time" + + "github.com/multiformats/go-multicodec" ) type ( @@ -14,8 +16,7 @@ type ( httpClient *http.Client httpClientTimeout time.Duration httpUserAgent string - ipfsDhtCascade bool - legacyCascade bool + protocols []multicodec.Code } ) @@ -26,8 +27,6 @@ func newOptions(o ...Option) (*options, error) { httpClient: http.DefaultClient, httpClientTimeout: time.Minute, httpUserAgent: "lassie", - ipfsDhtCascade: true, - legacyCascade: true, } for _, apply := range o { if err := apply(&opts); err != nil { @@ -48,7 +47,7 @@ func newOptions(o ...Option) (*options, error) { return &opts, nil } -// WithHttpClient sets the http.Client used to contact the indexer. +// WithHttpClient sets the http.Client used to contact the delegated routing service. // Defaults to http.DefaultClient if unspecified. func WithHttpClient(c *http.Client) Option { return func(o *options) error { @@ -57,7 +56,7 @@ func WithHttpClient(c *http.Client) Option { } } -// WithHttpClientTimeout sets the timeout for the HTTP requests sent to the indexer. +// WithHttpClientTimeout sets the timeout for the HTTP requests sent to the delegated routing service. // Defaults to one minute if unspecified. func WithHttpClientTimeout(t time.Duration) Option { return func(o *options) error { @@ -66,7 +65,7 @@ func WithHttpClientTimeout(t time.Duration) Option { } } -// WithHttpEndpoint sets the indexer HTTP API endpoint. +// WithHttpEndpoint sets the delegated routing HTTP API endpoint. // Defaults to https://cid.contact if unspecified. func WithHttpEndpoint(e *url.URL) Option { return func(o *options) error { @@ -75,7 +74,7 @@ func WithHttpEndpoint(e *url.URL) Option { } } -// WithHttpUserAgent sets the User-Agent header value when contacting the indexer HTTP API. +// WithHttpUserAgent sets the User-Agent header value when contacting the delegated routing HTTP API. // Setting this option to empty string will disable inclusion of User-Agent header. // Defaults to "lassie" if unspecified. func WithHttpUserAgent(a string) Option { @@ -94,20 +93,13 @@ func WithAsyncResultsChanBuffer(i int) Option { } } -// WithIpfsDhtCascade sets weather to cascade lookups onto the IPFS DHT. -// Enabled by default if unspecified. -func WithIpfsDhtCascade(b bool) Option { - return func(o *options) error { - o.ipfsDhtCascade = b - return nil - } -} - -// WithLegacyCascade sets weather to cascade finds legacy providers connecting only over bitswap -// Enabled by default if unspecified. -func WithLegacyCascade(b bool) Option { +// WithProtocols sets the list of protocols to filter by when querying the delegated routing service. +// This adds a filter-protocols query parameter to requests (for spec-compliant servers) +// and performs client-side filtering as fallback (for non-compliant servers like cid.contact). +// If unspecified, no protocol filtering is applied. +func WithProtocols(protocols []multicodec.Code) Option { return func(o *options) error { - o.legacyCascade = b + o.protocols = protocols return nil } } diff --git a/pkg/internal/itest/linksystemutil/linksystemblockstore.go b/pkg/internal/itest/linksystemutil/linksystemblockstore.go new file mode 100644 index 00000000..9a93223d --- /dev/null +++ b/pkg/internal/itest/linksystemutil/linksystemblockstore.go @@ -0,0 +1,89 @@ +package linksystemutil + +import ( + "bytes" + "context" + "errors" + "io" + + "github.com/ipfs/boxo/blockstore" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +var _ blockstore.Blockstore = (*LinkSystemBlockstore)(nil) + +type LinkSystemBlockstore struct { + lsys linking.LinkSystem +} + +func NewLinkSystemBlockstore(lsys linking.LinkSystem) *LinkSystemBlockstore { + return &LinkSystemBlockstore{lsys} +} + +func (lsbs *LinkSystemBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error { + return errors.New("not supported") +} + +func (lsbs *LinkSystemBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { + _, err := lsbs.lsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err != nil { + return false, err + } + return true, nil +} + +func (lsbs *LinkSystemBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + rdr, err := lsbs.lsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err != nil { + return nil, err + } + var buf bytes.Buffer + _, err = io.Copy(&buf, rdr) + if err != nil { + return nil, err + } + return blocks.NewBlockWithCid(buf.Bytes(), c) +} + +func (lsbs *LinkSystemBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + rdr, err := lsbs.lsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err != nil { + return 0, err + } + i, err := io.Copy(io.Discard, rdr) + if err != nil { + return 0, err + } + return int(i), nil +} + +func (lsbs *LinkSystemBlockstore) Put(ctx context.Context, blk blocks.Block) error { + w, wc, err := lsbs.lsys.StorageWriteOpener(linking.LinkContext{Ctx: ctx}) + if err != nil { + return err + } + if _, err = io.Copy(w, bytes.NewReader(blk.RawData())); err != nil { + return err + } + return wc(cidlink.Link{Cid: blk.Cid()}) +} + +func (lsbs *LinkSystemBlockstore) PutMany(ctx context.Context, blks []blocks.Block) error { + for _, blk := range blks { + if err := lsbs.Put(ctx, blk); err != nil { + return err + } + } + return nil +} + +func (lsbs *LinkSystemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return nil, errors.New("not supported") +} + +func (lsbs *LinkSystemBlockstore) HashOnRead(enabled bool) { + lsbs.lsys.TrustedStorage = !enabled +} diff --git a/pkg/internal/itest/util.go b/pkg/internal/itest/util.go new file mode 100644 index 00000000..4eb5ff77 --- /dev/null +++ b/pkg/internal/itest/util.go @@ -0,0 +1,24 @@ +package itest + +import ( + "io" + "testing" + + "github.com/ipfs/go-unixfsnode" + "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/stretchr/testify/require" +) + +// CarBytesLinkSystem returns a LinkSystem based on a CAR body, it is a simple +// utility wrapper around the go-car storage Readable that is UnixFS aware. +func CarBytesLinkSystem(t *testing.T, carReader io.ReaderAt) ipld.LinkSystem { + reader, err := storage.OpenReadable(carReader) + require.NoError(t, err) + linkSys := cidlink.DefaultLinkSystem() + linkSys.SetReadStorage(reader) + linkSys.NodeReifier = unixfsnode.Reify + linkSys.TrustedStorage = true + return linkSys +} diff --git a/pkg/internal/lp2ptransports/lp2ptransports.go b/pkg/internal/lp2ptransports/lp2ptransports.go deleted file mode 100644 index 84c75204..00000000 --- a/pkg/internal/lp2ptransports/lp2ptransports.go +++ /dev/null @@ -1,62 +0,0 @@ -package lp2ptransports - -import ( - "context" - "fmt" - "time" - - "github.com/ipfs/go-log/v2" - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/protocol" -) - -var logger = log.Logger("lassie/lp2p/tspt/client") - -// TransportsProtocolID is the protocol for querying which retrieval transports -// the Storage Provider supports (http, libp2p, etc) -const TransportsProtocolID = protocol.ID("/fil/retrieval/transports/1.0.0") - -const streamReadDeadline = 5 * time.Second - -// TransportsClient sends retrieval queries over libp2p -type TransportsClient struct { - h host.Host -} - -// NewTransportsClient creates a new query over libp2p -func NewTransportsClient(h host.Host) *TransportsClient { - c := &TransportsClient{ - h: h, - } - return c -} - -// SendQuery sends a retrieval query over a libp2p stream to the peer -func (c *TransportsClient) SendQuery(ctx context.Context, id peer.ID) (*QueryResponse, error) { - logger.Debugw("query", "peer", id) - - // Create a libp2p stream to the provider - s, err := c.h.NewStream(ctx, id, TransportsProtocolID) - if err != nil { - return nil, err - } - - defer s.Close() // nolint - - // Set a deadline on reading from the stream so it doesn't hang - _ = s.SetReadDeadline(time.Now().Add(streamReadDeadline)) - defer s.SetReadDeadline(time.Time{}) // nolint - - // Read the response from the stream - queryResponsei, err := BindnodeRegistry.TypeFromReader(s, (*QueryResponse)(nil), dagcbor.Decode) - if err != nil { - return nil, fmt.Errorf("reading query response: %w", err) - } - queryResponse := queryResponsei.(*QueryResponse) - - logger.Debugw("response", "peer", id) - - return queryResponse, nil -} diff --git a/pkg/internal/lp2ptransports/types.go b/pkg/internal/lp2ptransports/types.go deleted file mode 100644 index bd85f05d..00000000 --- a/pkg/internal/lp2ptransports/types.go +++ /dev/null @@ -1,53 +0,0 @@ -package lp2ptransports - -import ( - _ "embed" - "fmt" - - "github.com/ipld/go-ipld-prime/node/bindnode" - bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry" - "github.com/multiformats/go-multiaddr" -) - -type Protocol struct { - // The name of the transport protocol eg "libp2p" or "http" - Name string - // The address of the endpoint in multiaddr format - Addresses []multiaddr.Multiaddr -} - -type QueryResponse struct { - Protocols []Protocol -} - -//go:embed types.ipldsch -var embedSchema []byte - -func multiAddrFromBytes(b []byte) (interface{}, error) { - ma, err := multiaddr.NewMultiaddrBytes(b) - if err != nil { - return nil, err - } - return &ma, err -} - -func multiAddrToBytes(iface interface{}) ([]byte, error) { - ma, ok := iface.(*multiaddr.Multiaddr) - if !ok { - return nil, fmt.Errorf("expected *Multiaddr value") - } - - return (*ma).Bytes(), nil -} - -var BindnodeRegistry = bindnoderegistry.NewRegistry() - -func init() { - var dummyMa multiaddr.Multiaddr - var bindnodeOptions = []bindnode.Option{ - bindnode.TypedBytesConverter(&dummyMa, multiAddrFromBytes, multiAddrToBytes), - } - if err := BindnodeRegistry.RegisterType((*QueryResponse)(nil), string(embedSchema), "QueryResponse", bindnodeOptions...); err != nil { - panic(err.Error()) - } -} diff --git a/pkg/internal/lp2ptransports/types.ipldsch b/pkg/internal/lp2ptransports/types.ipldsch deleted file mode 100644 index efa9271e..00000000 --- a/pkg/internal/lp2ptransports/types.ipldsch +++ /dev/null @@ -1,15 +0,0 @@ -# Defines the response to a query asking which transport protocols a -# Storage Provider supports -type Multiaddr bytes - -type Protocol struct { - # The name of the transport protocol - # Known protocols: "libp2p", "http", "https", "bitswap" - Name String - # The addresses of the endpoint in multiaddr format - Addresses [Multiaddr] -} - -type QueryResponse struct { - Protocols [Protocol] -} diff --git a/pkg/internal/mockindexer/mockindexer.go b/pkg/internal/mockindexer/mockindexer.go index f319c060..878bd9fc 100644 --- a/pkg/internal/mockindexer/mockindexer.go +++ b/pkg/internal/mockindexer/mockindexer.go @@ -2,6 +2,7 @@ package mockindexer import ( "context" + "encoding/base64" "encoding/json" "fmt" "net" @@ -10,8 +11,11 @@ import ( "time" "github.com/filecoin-project/go-clock" + "github.com/filecoin-project/lassie/pkg/indexerlookup" "github.com/ipfs/go-cid" "github.com/ipni/go-libipni/find/model" + "github.com/ipni/go-libipni/metadata" + "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" ) @@ -66,6 +70,7 @@ func NewMockIndexer( // Routes mux.HandleFunc("/multihash/", httpServer.handleMultihash) + mux.HandleFunc("/routing/v1/providers/", httpServer.handleDelegatedRouting) return httpServer, nil } @@ -156,3 +161,104 @@ func (s *MockIndexer) handleMultihash(res http.ResponseWriter, req *http.Request s.clock.Sleep(1 * time.Second) } } + +func (s *MockIndexer) handleDelegatedRouting(res http.ResponseWriter, req *http.Request) { + if s.connectCh != nil { + s.connectCh <- req.URL.String() + } + + // Parse path: /routing/v1/providers/{cid} + urlPath := strings.Split(req.URL.Path, "/") + + // Expected: ["", "routing", "v1", "providers", "{cid}"] + if len(urlPath) < 5 { + res.WriteHeader(http.StatusNotFound) + return + } + + // Extract CID from path + cidStr := urlPath[4] + c, err := cid.Decode(cidStr) + if err != nil { + http.Error(res, "Failed to parse CID parameter", http.StatusBadRequest) + return + } + + // Look up providers by multihash + returnResults := s.returnedValues[string(c.Hash())] + if len(returnResults) == 0 { + http.NotFound(res, req) + return + } + + // Convert to delegated routing format + var providers []indexerlookup.DelegatedProvider + for _, result := range returnResults { + // Decode metadata to get protocols + md := metadata.Default.New() + if err := md.UnmarshalBinary(result.Metadata); err != nil { + continue // Skip providers with bad metadata + } + + // Build provider record + provider := indexerlookup.DelegatedProvider{ + Schema: "peer", + ID: result.Provider.ID.String(), + Addrs: make([]string, 0, len(result.Provider.Addrs)), + Protocols: make([]string, 0), + Metadata: make(map[string]interface{}), + } + + // Convert multiaddrs to strings + for _, addr := range result.Provider.Addrs { + provider.Addrs = append(provider.Addrs, addr.String()) + } + + // Convert protocols + for _, protoCode := range md.Protocols() { + var protoName string + switch protoCode { + case multicodec.TransportGraphsyncFilecoinv1: + protoName = "transport-graphsync-filecoinv1" + // Get the protocol-specific metadata + if proto := md.Get(protoCode); proto != nil { + if gs, ok := proto.(*metadata.GraphsyncFilecoinV1); ok { + // Encode as base64 for testing compatibility + if metadataBytes, err := proto.MarshalBinary(); err == nil { + // Remove the varint protocol ID prefix for the base64 encoding + // The metadata field should just contain the CBOR payload + provider.Metadata[protoName] = base64.StdEncoding.EncodeToString(metadataBytes) + } else { + // Fallback: provide as JSON object + provider.Metadata[protoName] = map[string]interface{}{ + "PieceCID": gs.PieceCID.String(), + "VerifiedDeal": gs.VerifiedDeal, + "FastRetrieval": gs.FastRetrieval, + } + } + } + } + case multicodec.TransportIpfsGatewayHttp: + protoName = "transport-ipfs-gateway-http" + provider.Metadata[protoName] = "" // No additional metadata + default: + continue // Skip unknown protocols + } + provider.Protocols = append(provider.Protocols, protoName) + } + + providers = append(providers, provider) + } + + // Build response + response := indexerlookup.DelegatedRoutingResponse{ + Providers: providers, + } + + // Send JSON response + res.Header().Set("Content-Type", "application/json") + encoder := json.NewEncoder(res) + if err := encoder.Encode(response); err != nil { + http.Error(res, err.Error(), http.StatusInternalServerError) + } +} diff --git a/pkg/internal/testutil/collectingeventlsubscriber.go b/pkg/internal/testutil/collectingeventlsubscriber.go index 2bf82b03..9eaa4756 100644 --- a/pkg/internal/testutil/collectingeventlsubscriber.go +++ b/pkg/internal/testutil/collectingeventlsubscriber.go @@ -81,12 +81,12 @@ func VerifyContainsCollectedEvent(t *testing.T, afterStart time.Duration, expect if actual.Code() == expected.Code() && actual.RetrievalId() == expected.RetrievalId() { - actualSpEvent, aspOk := actual.(events.EventWithProviderID) - expectedSpEvent, espOk := expected.(events.EventWithProviderID) - haveMatchingSpIDs := aspOk && espOk && expectedSpEvent.ProviderId() == actualSpEvent.ProviderId() - haveNoSpIDs := !aspOk && !espOk + actualEpEvent, aepOk := actual.(events.EventWithEndpoint) + expectedEpEvent, eepOk := expected.(events.EventWithEndpoint) + haveMatchingEndpoints := aepOk && eepOk && expectedEpEvent.Endpoint() == actualEpEvent.Endpoint() + haveNoEndpoints := !aepOk && !eepOk - if haveMatchingSpIDs || haveNoSpIDs { + if haveMatchingEndpoints || haveNoEndpoints { VerifyCollectedEvent(t, actual, expected) return actual.Code() } @@ -102,12 +102,12 @@ func VerifyCollectedEvent(t *testing.T, actual types.RetrievalEvent, expected ty require.Equal(t, expected.RootCid(), actual.RootCid(), fmt.Sprintf("cid for %s", expected.Code())) require.Equal(t, expected.Time(), actual.Time(), fmt.Sprintf("time for %s", expected.Code())) - if asp, ok := actual.(events.EventWithProviderID); ok { - esp, ok := expected.(events.EventWithProviderID) + if aep, ok := actual.(events.EventWithEndpoint); ok { + eep, ok := expected.(events.EventWithEndpoint) if !ok { - require.Fail(t, fmt.Sprintf("expected event %s should be EventWithSPID", expected.Code())) + require.Fail(t, fmt.Sprintf("expected event %s should be EventWithEndpoint", expected.Code())) } - require.Equal(t, esp.ProviderId().String(), asp.ProviderId().String(), fmt.Sprintf("storage provider id for %s", expected.Code())) + require.Equal(t, eep.Endpoint(), aep.Endpoint(), fmt.Sprintf("endpoint for %s", expected.Code())) } if ec, ok := expected.(events.EventWithCandidates); ok { if ac, ok := actual.(events.EventWithCandidates); ok { diff --git a/pkg/internal/testutil/gen.go b/pkg/internal/testutil/gen.go index 3a26593e..1607b939 100644 --- a/pkg/internal/testutil/gen.go +++ b/pkg/internal/testutil/gen.go @@ -3,7 +3,7 @@ package testutil import ( "fmt" "io" - "math/rand" + "math/rand/v2" "net" "strconv" "testing" @@ -34,9 +34,11 @@ var seedSeq int64 // RandomBytes returns a byte array of the given size with random values. func RandomBytes(n int64) []byte { data := make([]byte, n) - src := rand.NewSource(seedSeq) + var seed [32]byte + seed[0] = byte(seedSeq) + seed[1] = byte(seedSeq >> 8) seedSeq++ - r := rand.New(src) + r := rand.NewChaCha8(seed) _, _ = r.Read(data) return data } @@ -69,9 +71,11 @@ func GenerateCids(n int) []cid.Cid { // GeneratePeers creates n peer ids. func GeneratePeers(t *testing.T, n int) []peer.ID { - src := rand.NewSource(seedSeq) + var seed [32]byte + seed[0] = byte(seedSeq) + seed[1] = byte(seedSeq >> 8) seedSeq++ - r := rand.New(src) + r := rand.NewChaCha8(seed) peerIds := make([]peer.ID, 0, n) for i := 0; i < n; i++ { _, publicKey, err := crypto.GenerateEd25519Key(r) @@ -109,7 +113,7 @@ func GenerateRetrievalCandidatesForCID(t *testing.T, n int, c cid.Cid, protocols candidates := make([]types.RetrievalCandidate, 0, n) peers := GeneratePeers(t, n) if len(protocols) == 0 { - protocols = []metadata.Protocol{&metadata.Bitswap{}} + protocols = []metadata.Protocol{metadata.IpfsGatewayHttp{}} } for i := 0; i < n; i++ { addrs := []multiaddr.Multiaddr{GenerateHTTPMultiAddr()} @@ -120,7 +124,7 @@ func GenerateRetrievalCandidatesForCID(t *testing.T, n int, c cid.Cid, protocols func GenerateMultiAddr() multiaddr.Multiaddr { // generate a random ipv4 address - addr := &net.TCPAddr{IP: net.IPv4(byte(rand.Intn(255)), byte(rand.Intn(255)), byte(rand.Intn(255)), byte(rand.Intn(255))), Port: rand.Intn(65535)} + addr := &net.TCPAddr{IP: net.IPv4(byte(rand.IntN(255)), byte(rand.IntN(255)), byte(rand.IntN(255)), byte(rand.IntN(255))), Port: rand.IntN(65535)} maddr, err := manet.FromIP(addr.IP) if err != nil { panic(err) diff --git a/pkg/internal/testutil/mocksession.go b/pkg/internal/testutil/mocksession.go index 4d2349de..37dda317 100644 --- a/pkg/internal/testutil/mocksession.go +++ b/pkg/internal/testutil/mocksession.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/filecoin-project/lassie/pkg/session" "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" @@ -32,8 +31,6 @@ type SessionMetric struct { type MockSession struct { ctx context.Context - actual *session.Session - providerTimeout time.Duration blockList map[peer.ID]bool candidatePreferenceOrder []types.RetrievalCandidate metricsCh chan SessionMetric @@ -46,21 +43,10 @@ func NewMockSession(ctx context.Context) *MockSession { } } -// WithActual sets a real session to be used for all methods where no value is -// currently set. The primary use of this is to test session implementation -// functionality while collecting call information. -func (ms *MockSession) WithActual(session *session.Session) { - ms.actual = session -} - func (ms *MockSession) SetCandidatePreferenceOrder(candidatePreferenceOrder []types.RetrievalCandidate) { ms.candidatePreferenceOrder = candidatePreferenceOrder } -func (ms *MockSession) SetProviderTimeout(providerTimeout time.Duration) { - ms.providerTimeout = providerTimeout -} - func (ms *MockSession) SetBlockList(blockList map[peer.ID]bool) { ms.blockList = blockList } @@ -78,46 +64,24 @@ func (ms *MockSession) VerifyMetricsAt(ctx context.Context, t *testing.T, afterS require.ElementsMatch(t, expectedMetrics, metricsReceived) } -func (ms *MockSession) GetStorageProviderTimeout(storageProviderId peer.ID) time.Duration { - if ms.actual != nil && ms.providerTimeout == 0 { - return ms.actual.GetStorageProviderTimeout(storageProviderId) - } - return ms.providerTimeout -} - func (ms *MockSession) FilterIndexerCandidate(candidate types.RetrievalCandidate) (bool, types.RetrievalCandidate) { - if ms.actual != nil && len(ms.blockList) == 0 { - return ms.actual.FilterIndexerCandidate(candidate) - } blocked := ms.blockList[candidate.MinerPeer.ID] return !blocked, candidate } func (ms *MockSession) RegisterRetrieval(retrievalId types.RetrievalID, cid cid.Cid, selector datamodel.Node) bool { - if ms.actual != nil { - return ms.actual.RegisterRetrieval(retrievalId, cid, selector) - } return true } func (ms *MockSession) AddToRetrieval(retrievalId types.RetrievalID, storageProviderIds []peer.ID) error { - if ms.actual != nil { - return ms.actual.AddToRetrieval(retrievalId, storageProviderIds) - } return nil } func (ms *MockSession) EndRetrieval(retrievalId types.RetrievalID) error { - if ms.actual != nil { - return ms.actual.EndRetrieval(retrievalId) - } return nil } func (ms *MockSession) RecordConnectTime(storageProviderId peer.ID, connectTime time.Duration) { - if ms.actual != nil { - ms.actual.RecordConnectTime(storageProviderId, connectTime) - } ms.addMetric(SessionMetric{ Type: SessionMetric_Connect, Provider: storageProviderId, @@ -126,9 +90,6 @@ func (ms *MockSession) RecordConnectTime(storageProviderId peer.ID, connectTime } func (ms *MockSession) RecordFirstByteTime(storageProviderId peer.ID, firstByteTime time.Duration) { - if ms.actual != nil { - ms.actual.RecordFirstByteTime(storageProviderId, firstByteTime) - } ms.addMetric(SessionMetric{ Type: SessionMetric_FirstByte, Provider: storageProviderId, @@ -137,11 +98,6 @@ func (ms *MockSession) RecordFirstByteTime(storageProviderId peer.ID, firstByteT } func (ms *MockSession) RecordFailure(retrievalId types.RetrievalID, storageProviderId peer.ID) error { - if ms.actual != nil { - if err := ms.actual.RecordFailure(retrievalId, storageProviderId); err != nil { - return err - } - } ms.addMetric(SessionMetric{ Type: SessionMetric_Failure, Provider: storageProviderId, @@ -150,9 +106,6 @@ func (ms *MockSession) RecordFailure(retrievalId types.RetrievalID, storageProvi } func (ms *MockSession) RecordSuccess(storageProviderId peer.ID, bandwidthBytesPerSecond uint64) { - if ms.actual != nil { - ms.actual.RecordSuccess(storageProviderId, bandwidthBytesPerSecond) - } ms.addMetric(SessionMetric{ Type: SessionMetric_Success, Provider: storageProviderId, @@ -160,9 +113,6 @@ func (ms *MockSession) RecordSuccess(storageProviderId peer.ID, bandwidthBytesPe }) } func (ms *MockSession) ChooseNextProvider(peers []peer.ID, metadata []metadata.Protocol) int { - if ms.actual != nil && len(ms.candidatePreferenceOrder) == 0 { - return ms.actual.ChooseNextProvider(peers, metadata) - } for _, candidate := range ms.candidatePreferenceOrder { for i, peer := range peers { if candidate.MinerPeer.ID == peer { diff --git a/pkg/lassie/lassie.go b/pkg/lassie/lassie.go index 7ab09631..62add89e 100644 --- a/pkg/lassie/lassie.go +++ b/pkg/lassie/lassie.go @@ -5,25 +5,18 @@ import ( "net/http" "time" + "github.com/filecoin-project/lassie/pkg/extractor" "github.com/filecoin-project/lassie/pkg/indexerlookup" - "github.com/filecoin-project/lassie/pkg/net/client" - "github.com/filecoin-project/lassie/pkg/net/host" "github.com/filecoin-project/lassie/pkg/retriever" "github.com/filecoin-project/lassie/pkg/session" "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/sync" - "github.com/libp2p/go-libp2p" + "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multicodec" ) var _ types.Fetcher = &Lassie{} -const DefaultProviderTimeout = 20 * time.Second -const DefaultBitswapConcurrency = 32 -const DefaultBitswapConcurrencyPerRetrieval = 12 - // Lassie represents a reusable retrieval client. type Lassie struct { cfg *LassieConfig @@ -32,17 +25,11 @@ type Lassie struct { // LassieConfig customizes the behavior of a Lassie instance. type LassieConfig struct { - Source types.CandidateSource - Host host.Host - ProviderTimeout time.Duration - ConcurrentSPRetrievals uint - GlobalTimeout time.Duration - Libp2pOptions []libp2p.Option - Protocols []multicodec.Code - ProviderBlockList map[peer.ID]bool - ProviderAllowList map[peer.ID]bool - BitswapConcurrency int - BitswapConcurrencyPerRetrieval int + Source types.CandidateSource + GlobalTimeout time.Duration + ProviderBlockList map[peer.ID]bool + ProviderAllowList map[peer.ID]bool + SkipBlockVerification bool } type LassieOption func(cfg *LassieConfig) @@ -65,29 +52,15 @@ func NewLassieConfig(opts ...LassieOption) *LassieConfig { // NewLassieWithConfig creates a new Lassie instance with a custom // configuration. func NewLassieWithConfig(ctx context.Context, cfg *LassieConfig) (*Lassie, error) { - if cfg.Source == nil { - var err error - cfg.Source, err = indexerlookup.NewCandidateSource(indexerlookup.WithHttpClient(&http.Client{})) - if err != nil { - return nil, err - } - } + // HTTP-only protocol + protocols := []multicodec.Code{multicodec.TransportIpfsGatewayHttp} - if cfg.ProviderTimeout == 0 { - cfg.ProviderTimeout = DefaultProviderTimeout - } - if cfg.BitswapConcurrency == 0 { - cfg.BitswapConcurrency = DefaultBitswapConcurrency - } - if cfg.BitswapConcurrencyPerRetrieval == 0 { - cfg.BitswapConcurrencyPerRetrieval = DefaultBitswapConcurrencyPerRetrieval - } - - datastore := sync.MutexWrap(datastore.NewMapDatastore()) - - if cfg.Host == nil { + if cfg.Source == nil { var err error - cfg.Host, err = host.InitHost(ctx, cfg.Libp2pOptions) + cfg.Source, err = indexerlookup.NewCandidateSource( + indexerlookup.WithHttpClient(&http.Client{}), + indexerlookup.WithProtocols(protocols), + ) if err != nil { return nil, err } @@ -95,54 +68,26 @@ func NewLassieWithConfig(ctx context.Context, cfg *LassieConfig) (*Lassie, error sessionConfig := session.DefaultConfig(). WithProviderBlockList(cfg.ProviderBlockList). - WithProviderAllowList(cfg.ProviderAllowList). - WithDefaultProviderConfig(session.ProviderConfig{ - RetrievalTimeout: cfg.ProviderTimeout, - MaxConcurrentRetrievals: cfg.ConcurrentSPRetrievals, - }) - session := session.NewSession(sessionConfig, true) - - if len(cfg.Protocols) == 0 { - cfg.Protocols = []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportGraphsyncFilecoinv1, multicodec.TransportIpfsGatewayHttp} - } - - protocolRetrievers := make(map[multicodec.Code]types.CandidateRetriever) - for _, protocol := range cfg.Protocols { - switch protocol { - case multicodec.TransportGraphsyncFilecoinv1: - retrievalClient, err := client.NewClient(ctx, datastore, cfg.Host) - if err != nil { - return nil, err - } - - if err := retrievalClient.AwaitReady(); err != nil { // wait for dt setup - return nil, err - } - protocolRetrievers[protocol] = retriever.NewGraphsyncRetriever(session, retrievalClient) - // DISABLED: bitswap support removed for boxo v0.35.0 compatibility - // case multicodec.TransportBitswap: - // protocolRetrievers[protocol] = retriever.NewBitswapRetrieverFromHost(ctx, cfg.Host, retriever.BitswapConfig{ - // BlockTimeout: cfg.ProviderTimeout, - // Concurrency: cfg.BitswapConcurrency, - // ConcurrencyPerRetrieval: cfg.BitswapConcurrencyPerRetrieval, - // }) - case multicodec.TransportIpfsGatewayHttp: - protocolRetrievers[protocol] = retriever.NewHttpRetriever(session, http.DefaultClient) - } - } + WithProviderAllowList(cfg.ProviderAllowList) + sess := session.NewSession(sessionConfig, true) - retriever, err := retriever.NewRetriever(ctx, session, cfg.Source, protocolRetrievers) + httpRetriever := retriever.NewHttpRetriever(sess, http.DefaultClient) + ret, err := retriever.NewRetriever(ctx, sess, cfg.Source, httpRetriever, multicodec.TransportIpfsGatewayHttp) if err != nil { return nil, err } - retriever.Start() - lassie := &Lassie{ + // Wrap the retriever with HybridRetriever for per-block fallback + ret.WrapWithHybrid(cfg.Source, http.DefaultClient, cfg.SkipBlockVerification) + + ret.Start() + + l := &Lassie{ cfg: cfg, - retriever: retriever, + retriever: ret, } - return lassie, nil + return l, nil } // WithCandidateSource allows you to specify a custom candidate finder. @@ -152,15 +97,6 @@ func WithCandidateSource(finder types.CandidateSource) LassieOption { } } -// WithProviderTimeout allows you to specify a custom timeout for retrieving -// data from a provider. Beyond this limit, when no data has been received, -// the retrieval will fail. -func WithProviderTimeout(timeout time.Duration) LassieOption { - return func(cfg *LassieConfig) { - cfg.ProviderTimeout = timeout - } -} - // WithGlobalTimeout allows you to specify a custom timeout for the entire // retrieval process. func WithGlobalTimeout(timeout time.Duration) LassieOption { @@ -169,36 +105,6 @@ func WithGlobalTimeout(timeout time.Duration) LassieOption { } } -// WithHost allows you to specify a custom libp2p host. -func WithHost(host host.Host) LassieOption { - return func(cfg *LassieConfig) { - cfg.Host = host - } -} - -// WithLibp2pOpts allows you to specify custom libp2p options. -func WithLibp2pOpts(libp2pOptions ...libp2p.Option) LassieOption { - return func(cfg *LassieConfig) { - cfg.Libp2pOptions = libp2pOptions - } -} - -// WithConcurrentSPRetrievals allows you to specify a custom number of -// concurrent retrievals from a single storage provider. -func WithConcurrentSPRetrievals(maxConcurrentSPRtreievals uint) LassieOption { - return func(cfg *LassieConfig) { - cfg.ConcurrentSPRetrievals = maxConcurrentSPRtreievals - } -} - -// WithProtocols allows you to specify a custom set of protocols to use for -// retrieval. -func WithProtocols(protocols []multicodec.Code) LassieOption { - return func(cfg *LassieConfig) { - cfg.Protocols = protocols - } -} - // WithProviderBlockList allows you to specify a custom provider block list. func WithProviderBlockList(providerBlockList map[peer.ID]bool) LassieOption { return func(cfg *LassieConfig) { @@ -215,21 +121,11 @@ func WithProviderAllowList(providerAllowList map[peer.ID]bool) LassieOption { } } -// WithBitswapConcurrency allows you to specify a custom concurrency for bitswap -// retrievals across all parallel retrievals in the same Lassie instance. This -// is applied using a preloader during traversals. The default is 32. -func WithBitswapConcurrency(concurrency int) LassieOption { - return func(cfg *LassieConfig) { - cfg.BitswapConcurrency = concurrency - } -} - -// WithBitswapConcurrencyPerRetrieval allows you to specify a custom concurrency -// for bitswap retrievals for each individual parallel retrieval. This is -// applied using a preloader during traversals. The default is 8. -func WithBitswapConcurrencyPerRetrieval(concurrency int) LassieOption { +// WithSkipBlockVerification disables per-block hash verification. +// WARNING: This is dangerous - malicious gateways can serve arbitrary data! +func WithSkipBlockVerification(skip bool) LassieOption { return func(cfg *LassieConfig) { - cfg.BitswapConcurrencyPerRetrieval = concurrency + cfg.SkipBlockVerification = skip } } @@ -251,3 +147,20 @@ func (l *Lassie) Fetch(ctx context.Context, request types.RetrievalRequest, opts func (l *Lassie) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func() { return l.retriever.RegisterSubscriber(subscriber) } + +// Extract retrieves content and extracts it directly to disk. +// This is memory-efficient: blocks are processed once and discarded. +func (l *Lassie) Extract( + ctx context.Context, + rootCid cid.Cid, + ext *extractor.Extractor, + eventsCallback func(types.RetrievalEvent), + onBlock func(int), +) (*types.RetrievalStats, error) { + var cancel context.CancelFunc + if l.cfg.GlobalTimeout != time.Duration(0) { + ctx, cancel = context.WithTimeout(ctx, l.cfg.GlobalTimeout) + defer cancel() + } + return l.retriever.RetrieveAndExtract(ctx, rootCid, ext, eventsCallback, onBlock) +} diff --git a/pkg/net/client/client.go b/pkg/net/client/client.go deleted file mode 100644 index 4cc0c18a..00000000 --- a/pkg/net/client/client.go +++ /dev/null @@ -1,379 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "time" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - dtchannelmonitor "github.com/filecoin-project/go-data-transfer/v2/channelmonitor" - dtimpl "github.com/filecoin-project/go-data-transfer/v2/impl" - dtnetwork "github.com/filecoin-project/go-data-transfer/v2/network" - dttransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" - "github.com/hannahhoward/go-pubsub/ready" - - "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/traversal/selector" - - retrievaltypes "github.com/filecoin-project/go-retrieval-types" - - "github.com/filecoin-project/go-state-types/abi" - - "github.com/filecoin-project/lassie/pkg/retriever" - "github.com/filecoin-project/lassie/pkg/types" - - "github.com/ipfs/go-datastore" - - graphsync "github.com/ipfs/go-graphsync/impl" - gsnetwork "github.com/ipfs/go-graphsync/network" - - "github.com/ipfs/go-log/v2" - - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -// Logging -var logger = log.Logger("lassie/client") -var tracer trace.Tracer = otel.Tracer("lassie") - -const RetrievalQueryProtocol = "/fil/retrieval/qry/1.0.0" - -// ChannelMonitorConfig defaults -const acceptTimeout = 24 * time.Hour -const completeTimeout = 40 * time.Minute -const maxConsecutiveRestarts = 15 -const restartBackoff = 20 * time.Second -const restartDebounce = 10 * time.Second - -// GraphSyncOpt defaults -const maxInProgressIncomingRequests = 200 -const maxInProgressIncomingRequestsPerPeer = 20 -const maxInProgressOutgoingRequests = 200 -const maxMemoryPerPeerResponder = 32 << 20 -const maxMemoryResponder = 8 << 30 -const maxTraversalLinks = 32 * (1 << 20) -const messageSendRetries = 2 -const sendMessageTimeout = 2 * time.Minute - -type RetrievalClient struct { - dataTransfer datatransfer.Manager - host host.Host - ready *ready.ReadyManager -} - -type Config struct { - ChannelMonitorConfig dtchannelmonitor.Config - Datastore datastore.Batching - GraphsyncOpts []graphsync.Option - Host host.Host - RetrievalConfigurer datatransfer.TransportConfigurer -} - -// Creates a new RetrievalClient -func NewClient(ctx context.Context, datastore datastore.Batching, host host.Host, opts ...func(*Config)) (*RetrievalClient, error) { - cfg := &Config{ - ChannelMonitorConfig: dtchannelmonitor.Config{ - AcceptTimeout: acceptTimeout, - RestartDebounce: restartDebounce, - RestartBackoff: restartBackoff, - MaxConsecutiveRestarts: maxConsecutiveRestarts, - CompleteTimeout: completeTimeout, - // RestartAckTimeout: 30 * time.Second, - - // Called when a restart completes successfully - //OnRestartComplete func(id datatransfer.ChannelID) - }, - Datastore: datastore, - GraphsyncOpts: []graphsync.Option{ - graphsync.MaxInProgressIncomingRequests(maxInProgressIncomingRequests), - graphsync.MaxInProgressOutgoingRequests(maxInProgressOutgoingRequests), - graphsync.MaxMemoryResponder(maxMemoryResponder), - graphsync.MaxMemoryPerPeerResponder(maxMemoryPerPeerResponder), - graphsync.MaxInProgressIncomingRequestsPerPeer(maxInProgressIncomingRequestsPerPeer), - graphsync.MessageSendRetries(messageSendRetries), - graphsync.SendMessageTimeout(sendMessageTimeout), - graphsync.MaxLinksPerIncomingRequests(maxTraversalLinks), - graphsync.MaxLinksPerOutgoingRequests(maxTraversalLinks), - }, - Host: host, - } - - for _, opt := range opts { - opt(cfg) - } - - return NewClientWithConfig(ctx, cfg) -} - -// Creates a new RetrievalClient with the given Config -func NewClientWithConfig(ctx context.Context, cfg *Config) (*RetrievalClient, error) { - - graphSync := graphsync.New(ctx, - gsnetwork.NewFromLibp2pHost(cfg.Host), - cidlink.DefaultLinkSystem(), - cfg.GraphsyncOpts..., - ).(*graphsync.GraphSync) - - dtNetwork := dtnetwork.NewFromLibp2pHost(cfg.Host) - dtTransport := dttransport.NewTransport(cfg.Host.ID(), graphSync) - - dtRestartConfig := dtimpl.ChannelRestartConfig(cfg.ChannelMonitorConfig) - - dataTransfer, err := dtimpl.NewDataTransfer(cfg.Datastore, dtNetwork, dtTransport, dtRestartConfig) - if err != nil { - return nil, err - } - - err = dataTransfer.RegisterVoucherType(retrievaltypes.DealProposalType, nil) - if err != nil { - return nil, err - } - - err = dataTransfer.RegisterVoucherType(retrievaltypes.DealPaymentType, nil) - if err != nil { - return nil, err - } - - if cfg.RetrievalConfigurer != nil { - if err := dataTransfer.RegisterTransportConfigurer(retrievaltypes.DealProposalType, cfg.RetrievalConfigurer); err != nil { - return nil, err - } - } - - ready := ready.NewReadyManager() - dataTransfer.OnReady(func(err error) { - ready.FireReady(err) - }) - - if err := dataTransfer.Start(ctx); err != nil { - return nil, err - } - go func() { - <-ctx.Done() - dataTransfer.Stop(context.Background()) - }() - - client := &RetrievalClient{ - dataTransfer: dataTransfer, - host: cfg.Host, - ready: ready, - } - - return client, nil -} - -func (rc *RetrievalClient) AwaitReady() error { - return rc.ready.AwaitReady() -} - -func (rc *RetrievalClient) Connect(ctx context.Context, peerAddr peer.AddrInfo) error { - return rc.host.Connect(ctx, peerAddr) -} - -func (rc *RetrievalClient) RetrieveFromPeer( - ctx context.Context, - linkSystem ipld.LinkSystem, - peerID peer.ID, - proposal *retrievaltypes.DealProposal, - sel ipld.Node, - maxBlocks uint64, - eventsCallback datatransfer.Subscriber, - gracefulShutdownRequested <-chan struct{}, -) (*types.RetrievalStats, error) { - logger.Infof("Starting retrieval with miner peer ID: %s", peerID) - - ctx, span := tracer.Start(ctx, "rcRetrieveContent") - defer span.End() - - if _, err := selector.CompileSelector(sel); err != nil { - return nil, fmt.Errorf("invalid selector: %w", err) - } - - // Stats - startTime := time.Now() - var timeToFirstByte time.Duration - totalPayment := abi.NewTokenAmount(0) - - rootCid := proposal.PayloadCID - - paymentRequired := !proposal.PricePerByte.IsZero() || !proposal.UnsealPrice.IsZero() - - if paymentRequired { - return nil, errors.New("paid retrieval is not supported") - } - - // The next nonce (incrementing unique ID starting from 0) for the next voucher - var nonce uint64 = 0 - - // dtRes receives either an error (failure) or nil (success) which is waited - // on and handled below before exiting the function - dtRes := make(chan error, 1) - - finish := func(err error) { - select { - case dtRes <- err: - default: - } - } - - dealID := proposal.ID - allBytesReceived := false - dealComplete := false - - eventsCb := func(event datatransfer.Event, state datatransfer.ChannelState) { - if eventsCallback != nil { - defer eventsCallback(event, state) - } - var receivedFirstByte bool - - switch event.Code { - case datatransfer.DataReceived: - receivedFirstByte = true - case datatransfer.Error: - finish(fmt.Errorf("datatransfer error: %s", event.Message)) - return - case datatransfer.CleanupComplete: - finish(nil) - return - case datatransfer.NewVoucherResult: - lastVoucher := state.LastVoucherResult() - resType, err := retrievaltypes.DealResponseFromNode(lastVoucher.Voucher) - if err != nil { - logger.Errorf("unexpected voucher result received: %s", err.Error()) - return - } - if len(resType.Message) != 0 { - logger.Debugf("Received deal response voucher result %s (%v): %s\n\t%+v", resType.Status, resType.Status, resType.Message, resType) - } else { - logger.Debugf("Received deal response voucher result %s (%v)\n\t%+v", resType.Status, resType.Status, resType) - } - - switch resType.Status { - case retrievaltypes.DealStatusAccepted: - logger.Info("Deal accepted") - case retrievaltypes.DealStatusFundsNeeded, retrievaltypes.DealStatusFundsNeededLastPayment: - finish(fmt.Errorf("provider requested payment")) - return - case retrievaltypes.DealStatusRejected: - finish(fmt.Errorf("deal rejected: %s", resType.Message)) - return - case retrievaltypes.DealStatusFundsNeededUnseal, retrievaltypes.DealStatusUnsealing: - finish(fmt.Errorf("data is sealed")) - return - case retrievaltypes.DealStatusCancelled: - finish(fmt.Errorf("deal cancelled: %s", resType.Message)) - return - case retrievaltypes.DealStatusErrored: - finish(fmt.Errorf("deal errored: %s", resType.Message)) - return - case retrievaltypes.DealStatusCompleted: - if allBytesReceived { - finish(nil) - return - } - dealComplete = true - } - case datatransfer.FinishTransfer: - if dealComplete { - finish(nil) - return - } - allBytesReceived = true - case datatransfer.DataReceivedProgress: - // First byte has been received - timeToFirstByte = time.Since(startTime) - receivedFirstByte = true - } - - name := datatransfer.Events[event.Code] - code := event.Code - msg := event.Message - blocksIndex := state.ReceivedCidsTotal() - totalReceived := state.Received() - if !receivedFirstByte { // || rc.logRetrievalProgressEvents { - logger.Debugw("retrieval event", "dealID", dealID, "rootCid", rootCid, "peerID", peerID, "name", name, "code", code, "message", msg, "blocksIndex", blocksIndex, "totalReceived", totalReceived) - } - } - - // Submit the retrieval deal proposal to the miner - proposalVoucher := retrievaltypes.BindnodeRegistry.TypeToNode(proposal) - chanid, err := rc.dataTransfer.OpenPullDataChannel( - ctx, - peerID, - datatransfer.TypedVoucher{Type: retrievaltypes.DealProposalType, Voucher: proposalVoucher}, - proposal.PayloadCID, - sel, - datatransfer.WithSubscriber(eventsCb), - datatransfer.WithTransportOptions( - dttransport.UseStore(linkSystem), - dttransport.MaxLinks(maxBlocks), - ), - ) - if err != nil { - // We could fail before a successful proposal - return nil, fmt.Errorf("%w: %s", retriever.ErrDealProposalFailed, err) - } - defer func() { - ctx, cancel := context.WithTimeout(context.TODO(), 1*time.Second) - defer cancel() - rc.dataTransfer.CloseDataTransferChannel(ctx, chanid) - }() - - // Wait for the retrieval to finish before exiting the function -awaitfinished: - for { - select { - case err := <-dtRes: - if err != nil { - return nil, fmt.Errorf("data transfer failed: %w", err) - } - - logger.Debugf("data transfer for retrieval complete") - break awaitfinished - case <-gracefulShutdownRequested: - go func() { - rc.dataTransfer.CloseDataTransferChannel(ctx, chanid) - }() - case <-ctx.Done(): - return nil, ctx.Err() - } - } - - // Confirm that we actually ended up with the root block we wanted, failure - // here indicates a data transfer error that was not properly reported - if _, err := linkSystem.StorageReadOpener(ipld.LinkContext{}, cidlink.Link{Cid: rootCid}); err != nil { - if nf, ok := err.(interface{ NotFound() bool }); !ok || !nf.NotFound() { - logger.Errorf("could not query block store: %w", err) - } - return nil, errors.New("data transfer failed: unconfirmed block transfer") - } - - // Compile the retrieval stats - - state, err := rc.dataTransfer.ChannelState(ctx, chanid) - if err != nil { - return nil, fmt.Errorf("could not get channel state: %w", err) - } - - duration := time.Since(startTime) - speed := uint64(float64(state.Received()) / duration.Seconds()) - - return &types.RetrievalStats{ - RootCid: rootCid, - StorageProviderId: state.OtherPeer(), - Size: state.Received(), - Blocks: uint64(state.ReceivedCidsTotal()), - Duration: duration, - AverageSpeed: speed, - TotalPayment: totalPayment, - NumPayments: int(nonce), - AskPrice: proposal.PricePerByte, - TimeToFirstByte: timeToFirstByte, - }, nil -} diff --git a/pkg/net/host/host.go b/pkg/net/host/host.go deleted file mode 100644 index 0a6580ed..00000000 --- a/pkg/net/host/host.go +++ /dev/null @@ -1,61 +0,0 @@ -package host - -import ( - "context" - - "github.com/filecoin-project/lassie/pkg/build" - "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/p2p/muxer/yamux" - "github.com/libp2p/go-libp2p/p2p/protocol/identify" - "github.com/libp2p/go-libp2p/p2p/security/noise" - tls "github.com/libp2p/go-libp2p/p2p/security/tls" - quic "github.com/libp2p/go-libp2p/p2p/transport/quic" - "github.com/libp2p/go-libp2p/p2p/transport/tcp" - "github.com/libp2p/go-libp2p/p2p/transport/websocket" - webtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport" - "github.com/multiformats/go-multiaddr" -) - -const ( - yamuxID = "/yamux/1.0.0" // The yamux protocol ID - yamuxAcceptBacklog = 512 // The number of streams that can be waiting to be accepted -) - -func InitHost(ctx context.Context, opts []libp2p.Option, listenAddrs ...multiaddr.Multiaddr) (Host, error) { - opts = append([]libp2p.Option{libp2p.Identity(nil), libp2p.ResourceManager(&network.NullResourceManager{})}, opts...) - if len(listenAddrs) > 0 { - opts = append([]libp2p.Option{libp2p.ListenAddrs(listenAddrs...)}, opts...) - } - // add transports - opts = append([]libp2p.Option{libp2p.Transport(tcp.NewTCPTransport, tcp.WithMetrics()), libp2p.Transport(websocket.New), libp2p.Transport(quic.NewTransport), libp2p.Transport(webtransport.New)}, opts...) - // add security - opts = append([]libp2p.Option{libp2p.Security(tls.ID, tls.New), libp2p.Security(noise.ID, noise.New)}, opts...) - - // add muxers - opts = append([]libp2p.Option{libp2p.Muxer(yamuxID, yamuxTransport())}, opts...) - - host, err := libp2p.New(opts...) - if err != nil { - return nil, err - } - - // Set the identify protocol user agent - idService, err := identify.NewIDService(host, identify.UserAgent(build.Version)) - if err != nil { - return nil, err - } - idService.Start() - - return host, nil -} - -func yamuxTransport() network.Multiplexer { - tpt := *yamux.DefaultTransport - tpt.AcceptBacklog = yamuxAcceptBacklog - return &tpt -} - -// Host is a type alias for libp2p host -type Host = host.Host diff --git a/pkg/retriever/assignablecandidatefinder_test.go b/pkg/retriever/assignablecandidatefinder_test.go deleted file mode 100644 index ac60bce8..00000000 --- a/pkg/retriever/assignablecandidatefinder_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package retriever_test - -import ( - "context" - "errors" - "fmt" - "testing" - "time" - - "github.com/filecoin-project/lassie/pkg/internal/testutil" - "github.com/filecoin-project/lassie/pkg/retriever" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipfs/go-cid" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - trustlessutils "github.com/ipld/go-trustless-utils" - "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/require" -) - -func TestAssignableCandidateFinder(t *testing.T) { - ctx := context.Background() - cid1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") - cid2 := cid.MustParse("bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk") - - testCases := []struct { - name string - candidateResults map[cid.Cid][]string - candidateError error - filteredPeers []string - fixedPeers map[cid.Cid][]string - expectedEvents map[cid.Cid][]types.EventCode - expectedCandidates map[cid.Cid][]string - expectedErrors map[cid.Cid]error - }{ - { - name: "successful candidates, no filtering", - candidateResults: map[cid.Cid][]string{ - cid1: {"fiz", "bang", "booz"}, - cid2: {"apples", "oranges", "cheese"}, - }, - expectedCandidates: map[cid.Cid][]string{ - cid1: {"fiz", "bang", "booz"}, - cid2: {"apples", "oranges", "cheese"}, - }, - expectedEvents: map[cid.Cid][]types.EventCode{ - cid1: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - cid2: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - }, - }, - { - name: "candidate finder error", - candidateError: errors.New("something went wrong"), - expectedErrors: map[cid.Cid]error{ - cid1: fmt.Errorf("could not get retrieval candidates for %s: %w", cid1, errors.New("something went wrong")), - cid2: fmt.Errorf("could not get retrieval candidates for %s: %w", cid2, errors.New("something went wrong")), - }, - expectedEvents: map[cid.Cid][]types.EventCode{ - cid1: {types.StartedFindingCandidatesCode, types.FailedCode}, - cid2: {types.StartedFindingCandidatesCode, types.FailedCode}, - }, - }, - { - name: "no candidates, indexer", - candidateResults: map[cid.Cid][]string{ - cid1: {}, - cid2: {"apples", "oranges", "cheese"}, - }, - expectedErrors: map[cid.Cid]error{ - cid1: retriever.ErrNoCandidates, - }, - expectedCandidates: map[cid.Cid][]string{ - cid2: {"apples", "oranges", "cheese"}, - }, - expectedEvents: map[cid.Cid][]types.EventCode{ - cid1: {types.StartedFindingCandidatesCode, types.FailedCode}, - cid2: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - }, - }, - { - name: "successful candidates, filtering", - candidateResults: map[cid.Cid][]string{ - cid1: {"fiz", "bang", "booz"}, - cid2: {"apples", "oranges", "cheese"}, - }, - filteredPeers: []string{"fiz", "apples"}, - expectedCandidates: map[cid.Cid][]string{ - cid1: {"bang", "booz"}, - cid2: {"oranges", "cheese"}, - }, - expectedEvents: map[cid.Cid][]types.EventCode{ - cid1: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - cid2: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - }, - }, - { - name: "no candidates, via filtering", - candidateResults: map[cid.Cid][]string{ - cid1: {"fiz", "bang", "booz"}, - cid2: {"apples", "oranges", "cheese"}, - }, - filteredPeers: []string{"fiz", "bang", "booz"}, - expectedErrors: map[cid.Cid]error{ - cid1: retriever.ErrNoCandidates, - }, - expectedCandidates: map[cid.Cid][]string{ - cid2: {"apples", "oranges", "cheese"}, - }, - expectedEvents: map[cid.Cid][]types.EventCode{ - cid1: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.FailedCode}, - cid2: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - }, - }, - { - name: "fixed peers", - candidateResults: map[cid.Cid][]string{ - cid1: {"fiz", "bang", "booz"}, - cid2: {"apples", "oranges", "cheese"}, - }, - expectedCandidates: map[cid.Cid][]string{ - cid1: {"double", "trouble"}, - cid2: {"super", "duper"}, - }, - fixedPeers: map[cid.Cid][]string{ - cid1: {"double", "trouble"}, - cid2: {"super", "duper"}, - }, - expectedEvents: map[cid.Cid][]types.EventCode{ - cid1: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - cid2: {types.StartedFindingCandidatesCode, types.CandidatesFoundCode, types.CandidatesFilteredCode}, - }, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - req := require.New(t) - ctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - allCandidateResults := make(map[cid.Cid][]types.RetrievalCandidate, len(testCase.candidateResults)) - for c, stringResults := range testCase.candidateResults { - candidateResults := make([]types.RetrievalCandidate, 0, len(stringResults)) - for _, stringResult := range stringResults { - candidateResults = append(candidateResults, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peer.ID(stringResult)}}) - } - allCandidateResults[c] = candidateResults - } - if testCase.fixedPeers == nil { - testCase.fixedPeers = make(map[cid.Cid][]string) - } - allProviders := make(map[cid.Cid][]types.Provider, len(testCase.fixedPeers)) - for c, stringResults := range testCase.fixedPeers { - providers := make([]types.Provider, 0, len(stringResults)) - for _, stringResult := range stringResults { - providers = append(providers, types.Provider{ - Peer: peer.AddrInfo{ID: peer.ID(stringResult)}, - Protocols: []metadata.Protocol{&metadata.GraphsyncFilecoinV1{}, &metadata.Bitswap{}, &metadata.IpfsGatewayHttp{}}, - }) - } - allProviders[c] = providers - } - candidateSource := testutil.NewMockCandidateSource(testCase.candidateError, allCandidateResults) - isAcceptableStorageProvider := func(candidate types.RetrievalCandidate) (bool, types.RetrievalCandidate) { - for _, filteredPeer := range testCase.filteredPeers { - if candidate.MinerPeer.ID == peer.ID(filteredPeer) { - return false, types.RetrievalCandidate{} - } - } - return true, candidate - } - receivedCandidates := make(map[cid.Cid][]string) - appendCandidates := func(cid cid.Cid, candidates []types.RetrievalCandidate) { - stringCandidates := make([]string, 0, len(candidates)) - for _, candidate := range candidates { - stringCandidates = append(stringCandidates, string(candidate.MinerPeer.ID)) - } - receivedCandidates[cid] = stringCandidates - } - receivedEvents := make(map[cid.Cid][]types.RetrievalEvent) - retrievalCollector := func(evt types.RetrievalEvent) { - receivedEvents[evt.RootCid()] = append(receivedEvents[evt.RootCid()], evt) - } - retrievalCandidateFinder := retriever.NewAssignableCandidateFinder(candidateSource, isAcceptableStorageProvider) - rid1, err := types.NewRetrievalID() - req.NoError(err) - receivedErrors := make(map[cid.Cid]error) - var candidates []types.RetrievalCandidate - candidateCollector := func(incoming []types.RetrievalCandidate) { - candidates = append(candidates, incoming...) - } - - err = retrievalCandidateFinder.FindCandidates(ctx, types.RetrievalRequest{ - RetrievalID: rid1, - Request: trustlessutils.Request{Root: cid1}, - LinkSystem: cidlink.DefaultLinkSystem(), - Providers: allProviders[cid1], - }, retrievalCollector, candidateCollector) - if err != nil { - receivedErrors[cid1] = err - } else { - appendCandidates(cid1, candidates) - } - rid2, err := types.NewRetrievalID() - req.NoError(err) - candidates = nil - err = retrievalCandidateFinder.FindCandidates(ctx, types.RetrievalRequest{ - RetrievalID: rid2, - Request: trustlessutils.Request{Root: cid2}, - LinkSystem: cidlink.DefaultLinkSystem(), - Providers: allProviders[cid2], - }, retrievalCollector, candidateCollector) - if err != nil { - receivedErrors[cid2] = err - } else { - appendCandidates(cid2, candidates) - } - expectedCandidates := testCase.expectedCandidates - if expectedCandidates == nil { - expectedCandidates = make(map[cid.Cid][]string) - } - req.Equal(expectedCandidates, receivedCandidates) - expectedErrors := testCase.expectedErrors - if expectedErrors == nil { - expectedErrors = make(map[cid.Cid]error) - } - req.Equal(expectedErrors, receivedErrors) - receivedCodes := make(map[cid.Cid][]types.EventCode, len(receivedEvents)) - for key, events := range receivedEvents { - receivedCodes[key] = make([]types.EventCode, 0, len(events)) - for _, event := range events { - receivedCodes[key] = append(receivedCodes[key], event.Code()) - } - } - req.Equal(testCase.expectedEvents, receivedCodes) - }) - } - -} diff --git a/pkg/retriever/combinators/asyncsplitter.go b/pkg/retriever/combinators/asyncsplitter.go deleted file mode 100644 index ad12246c..00000000 --- a/pkg/retriever/combinators/asyncsplitter.go +++ /dev/null @@ -1,89 +0,0 @@ -package combinators - -import ( - "context" - - "github.com/filecoin-project/lassie/pkg/types" -) - -// AsyncCandidateSplitter creates an splitter for a candidate stream from a simple function to split a set of candidates. -// Essentially, on each new candidate set from the stream, we run the splitter function, and feed the split candidates sets -// into the appropriate downstream candidate streams -type AsyncCandidateSplitter[T comparable] struct { - Keys []T - CandidateSplitter types.CandidateSplitter[T] -} - -// NewCandidateSplitterFn constructs a new candidate splitter for the given keys -// it's used to insurate -type NewCandidateSplitterFn[T comparable] func(keys []T) types.CandidateSplitter[T] - -var _ types.AsyncCandidateSplitter[int] = AsyncCandidateSplitter[int]{} - -// NewAsyncCandidateSplitter creates a new candidate splitter with the given passed in sync splitter constructor -func NewAsyncCandidateSplitter[T comparable](keys []T, newCandidateSplitter NewCandidateSplitterFn[T]) AsyncCandidateSplitter[T] { - return AsyncCandidateSplitter[T]{ - Keys: keys, - CandidateSplitter: newCandidateSplitter(keys), - } -} - -// SplitRetrievalRequest splits candidate streams for a given retrieval request -func (acs AsyncCandidateSplitter[T]) SplitRetrievalRequest(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.AsyncRetrievalSplitter[T] { - retrievalSplitter := acs.CandidateSplitter.SplitRetrievalRequest(ctx, request, events) - return asyncRetrievalSplitter[T]{ - AsyncCandidateSplitter: acs, - ctx: ctx, - retrievalSplitter: retrievalSplitter, - } -} - -type asyncRetrievalSplitter[T comparable] struct { - AsyncCandidateSplitter[T] - ctx context.Context - retrievalSplitter types.RetrievalSplitter[T] -} - -const splitBufferSize = 16 - -// SplitAsyncCandidates performs the work of actually splitting streams for a given request -func (ars asyncRetrievalSplitter[T]) SplitAsyncCandidates(asyncCandidates types.InboundAsyncCandidates) (map[T]types.InboundAsyncCandidates, <-chan error) { - incoming := make(map[T]types.InboundAsyncCandidates, len(ars.Keys)) - outgoing := make(map[T]types.OutboundAsyncCandidates, len(ars.Keys)) - for _, k := range ars.Keys { - incoming[k], outgoing[k] = types.MakeAsyncCandidates(splitBufferSize) - } - errChan := make(chan error, 1) - go func() { - defer func() { - for _, outgoingCandidateStream := range outgoing { - close(outgoingCandidateStream) - } - close(errChan) - }() - for { - hasCandidates, candidates, err := asyncCandidates.Next(ars.ctx) - if err != nil { - errChan <- err - return - } - if !hasCandidates { - errChan <- nil - return - } - splitCandidateSets, err := ars.retrievalSplitter.SplitCandidates(candidates) - if err != nil { - errChan <- err - return - } - for k, candidateSet := range splitCandidateSets { - err := outgoing[k].SendNext(ars.ctx, candidateSet) - if err != nil { - errChan <- err - return - } - } - } - }() - return incoming, errChan -} diff --git a/pkg/retriever/combinators/asyncsplitter_test.go b/pkg/retriever/combinators/asyncsplitter_test.go deleted file mode 100644 index 3be23db4..00000000 --- a/pkg/retriever/combinators/asyncsplitter_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package combinators_test - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/filecoin-project/lassie/pkg/internal/testutil" - "github.com/filecoin-project/lassie/pkg/retriever/combinators" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/stretchr/testify/require" -) - -func TestAsyncCandidateSplitter(t *testing.T) { - ctx := context.Background() - candidateSets := make([][]types.RetrievalCandidate, 0, 3) - for i := 0; i < 3; i++ { - candidateSets = append(candidateSets, testutil.GenerateRetrievalCandidates(t, (i+1)*2)) - } - testCases := []struct { - name string - keys []string - splitFunc splitFunc - expectedReceivedSets map[string][][]types.RetrievalCandidate - expectedErr error - }{ - { - name: "simple even split", - keys: []string{"even", "odd"}, - splitFunc: func(candidates []types.RetrievalCandidate) (map[string][]types.RetrievalCandidate, error) { - candidateMap := make(map[string][]types.RetrievalCandidate, 2) - for i, candidate := range candidates { - var key string - if i%2 == 0 { - key = "even" - } else { - key = "odd" - } - candidateMap[key] = append(candidateMap[key], candidate) - } - return candidateMap, nil - }, - expectedReceivedSets: map[string][][]types.RetrievalCandidate{ - "even": { - {candidateSets[0][0]}, - {candidateSets[1][0], candidateSets[1][2]}, - {candidateSets[2][0], candidateSets[2][2], candidateSets[2][4]}, - }, - "odd": { - {candidateSets[0][1]}, - {candidateSets[1][1], candidateSets[1][3]}, - {candidateSets[2][1], candidateSets[2][3], candidateSets[2][5]}, - }, - }, - }, - { - name: "stateful", - keys: []string{"batch1", "batch2"}, - splitFunc: (&statefulSplit{splitPoint: 4}).splitFunc, - expectedReceivedSets: map[string][][]types.RetrievalCandidate{ - "batch1": { - {candidateSets[0][0], candidateSets[0][1]}, - {candidateSets[1][0], candidateSets[1][1]}, - }, - "batch2": { - {candidateSets[1][2], candidateSets[1][3]}, - {candidateSets[2][0], - candidateSets[2][1], - candidateSets[2][2], - candidateSets[2][3], - candidateSets[2][4], - candidateSets[2][5]}, - }, - }, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - req := require.New(t) - ctx, cancel := context.WithTimeout(ctx, 3*time.Second) - defer cancel() - var wg sync.WaitGroup - incoming, outgoing := types.MakeAsyncCandidates(0) - wg.Add(1) - go func() { - defer wg.Done() - for _, candidates := range candidateSets { - req.NoError(outgoing.SendNext(ctx, candidates)) - } - close(outgoing) - }() - asyncCandidateSplitter := combinators.NewAsyncCandidateSplitter(testCase.keys, func([]string) types.CandidateSplitter[string] { - return mockSplitter{testCase.splitFunc} - }) - asyncRetrievalSplitter := asyncCandidateSplitter.SplitRetrievalRequest(ctx, testutil.GenerateRetrievalRequests(t, 1)[0], func(types.RetrievalEvent) {}) - streams, errChan := asyncRetrievalSplitter.SplitAsyncCandidates(incoming) - req.Len(streams, len(testCase.keys)) - for _, key := range testCase.keys { - key := key - stream, ok := streams[key] - req.True(ok) - wg.Add(1) - go func() { - defer wg.Done() - for _, candidates := range testCase.expectedReceivedSets[key] { - hasCandidates, nextCandidates, err := stream.Next(ctx) - req.NoError(err) - req.True(hasCandidates) - req.Equal(candidates, nextCandidates) - } - hasCandidates, nextCandidates, err := stream.Next(ctx) - req.NoError(err) - req.False(hasCandidates) - req.Nil(nextCandidates) - }() - } - wg.Add(1) - go func() { - defer wg.Done() - select { - case <-ctx.Done(): - req.FailNow("did not receive on error channel") - case err := <-errChan: - if testCase.expectedErr != nil { - req.EqualError(err, testCase.expectedErr.Error()) - } else { - req.NoError(err) - } - } - }() - wg.Wait() - }) - } -} - -type splitFunc func([]types.RetrievalCandidate) (map[string][]types.RetrievalCandidate, error) - -type mockSplitter struct { - splitFunc splitFunc -} - -func (ms mockSplitter) SplitRetrievalRequest(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.RetrievalSplitter[string] { - return ms -} - -func (ms mockSplitter) SplitCandidates(candidates []types.RetrievalCandidate) (map[string][]types.RetrievalCandidate, error) { - return ms.splitFunc(candidates) -} - -type statefulSplit struct { - splitPoint int - receivedSoFar int -} - -func (ss *statefulSplit) splitFunc(candidates []types.RetrievalCandidate) (map[string][]types.RetrievalCandidate, error) { - candidateMap := make(map[string][]types.RetrievalCandidate, 2) - for _, candidate := range candidates { - var key string - if ss.receivedSoFar >= ss.splitPoint { - key = "batch2" - } else { - key = "batch1" - } - ss.receivedSoFar++ - candidateMap[key] = append(candidateMap[key], candidate) - } - return candidateMap, nil -} diff --git a/pkg/retriever/combinators/doc.go b/pkg/retriever/combinators/doc.go deleted file mode 100644 index 95f6b068..00000000 --- a/pkg/retriever/combinators/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -/* -Package combinators contains tools to put various retrieval components together to make -full retrievers -*/ -package combinators diff --git a/pkg/retriever/combinators/retrieverwithcandidatefinder.go b/pkg/retriever/combinators/retrieverwithcandidatefinder.go deleted file mode 100644 index 3e9d0586..00000000 --- a/pkg/retriever/combinators/retrieverwithcandidatefinder.go +++ /dev/null @@ -1,53 +0,0 @@ -package combinators - -import ( - "context" - - "github.com/filecoin-project/lassie/pkg/types" -) - -// RetrieverWithCandidateFinder retrieves from a candidate retriever after first retrieving candidates from a candidate finder -type RetrieverWithCandidateFinder struct { - CandidateFinder types.CandidateFinder - CandidateRetriever types.CandidateRetriever -} - -var _ types.Retriever = RetrieverWithCandidateFinder{} - -func (rcf RetrieverWithCandidateFinder) Retrieve(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - asyncCandidateRetrieval := rcf.CandidateRetriever.Retrieve(ctx, request, events) - - asyncCandidates := make(chan []types.RetrievalCandidate) - outbound := types.OutboundAsyncCandidates(asyncCandidates) - inbound := types.InboundAsyncCandidates(asyncCandidates) - - findErr := make(chan error, 1) - resultChan := make(chan types.RetrievalResult, 1) - go func(findErr chan<- error) { - defer close(findErr) - defer close(outbound) - err := rcf.CandidateFinder.FindCandidates(ctx, request, events, func(candidates []types.RetrievalCandidate) { - _ = outbound.SendNext(ctx, candidates) - }) - findErr <- err - }(findErr) - go func() { - stats, err := asyncCandidateRetrieval.RetrieveFromAsyncCandidates(inbound) - resultChan <- types.RetrievalResult{Stats: stats, Err: err} - }() - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case err := <-findErr: - if err != nil { - return nil, err - } - findErr = nil - case result := <-resultChan: - return result.Stats, result.Err - } - } -} diff --git a/pkg/retriever/combinators/splitretriever.go b/pkg/retriever/combinators/splitretriever.go deleted file mode 100644 index 2fdb190f..00000000 --- a/pkg/retriever/combinators/splitretriever.go +++ /dev/null @@ -1,67 +0,0 @@ -package combinators - -import ( - "context" - "errors" - - "github.com/filecoin-project/lassie/pkg/retriever/coordinators" - "github.com/filecoin-project/lassie/pkg/types" -) - -var _ types.CandidateRetriever = SplitRetriever[int]{} - -// SplitRetriever retrieves by first splitting candidates with a splitter, then passing the split sets -// to multiple retrievers using the given coordination style -type SplitRetriever[T comparable] struct { - AsyncCandidateSplitter types.AsyncCandidateSplitter[T] - CandidateRetrievers map[T]types.CandidateRetriever - CoordinationKind types.CoordinationKind -} - -func (m SplitRetriever[T]) Retrieve(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.CandidateRetrieval { - - return splitRetrieval[T]{ - retrievalSplitter: m.AsyncCandidateSplitter.SplitRetrievalRequest(ctx, request, events), - candidateRetrievers: m.CandidateRetrievers, - coordinationKind: m.CoordinationKind, - ctx: ctx, - request: request, - events: events, - } -} - -type splitRetrieval[T comparable] struct { - retrievalSplitter types.AsyncRetrievalSplitter[T] - candidateRetrievers map[T]types.CandidateRetriever - coordinationKind types.CoordinationKind - ctx context.Context - request types.RetrievalRequest - events func(types.RetrievalEvent) -} - -func (m splitRetrieval[T]) RetrieveFromAsyncCandidates(asyncCandidates types.InboundAsyncCandidates) (*types.RetrievalStats, error) { - asyncSplitCandidates, errChan := m.retrievalSplitter.SplitAsyncCandidates(asyncCandidates) - coordinator, err := coordinators.Coordinator(m.coordinationKind) - if err != nil { - return nil, err - } - stats, err := coordinator(m.ctx, func(ctx context.Context, retrievalCall func(types.RetrievalTask)) { - candidateRetrievals := make(map[T]types.CandidateRetrieval, len(m.candidateRetrievers)) - for key, candidateRetriever := range m.candidateRetrievers { - candidateRetrievals[key] = candidateRetriever.Retrieve(ctx, m.request, m.events) - } - for key, asyncCandidates := range asyncSplitCandidates { - if asyncCandidateRetrieval, ok := candidateRetrievals[key]; ok { - retrievalCall(types.AsyncRetrievalTask{ - Candidates: asyncCandidates, - AsyncCandidateRetrieval: asyncCandidateRetrieval, - }) - } - } - retrievalCall(types.DeferredErrorTask{Ctx: ctx, ErrChan: errChan}) - }) - if stats == nil && err == nil { - return nil, errors.New("no eligible retrievers") - } - return stats, err -} diff --git a/pkg/retriever/coordinators/doc.go b/pkg/retriever/coordinators/doc.go deleted file mode 100644 index feae0447..00000000 --- a/pkg/retriever/coordinators/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Package coordinators contains retrievers that coordinate multiple child retrievers -*/ -package coordinators - -import ( - "errors" - - "github.com/filecoin-project/lassie/pkg/types" -) - -func Coordinator(kind types.CoordinationKind) (types.RetrievalCoordinator, error) { - switch kind { - case types.RaceCoordination: - return Race, nil - case types.SequentialCoordination: - return Sequence, nil - default: - return nil, errors.New("unrecognized retriever kind") - } -} diff --git a/pkg/retriever/coordinators/race.go b/pkg/retriever/coordinators/race.go deleted file mode 100644 index f1d1676a..00000000 --- a/pkg/retriever/coordinators/race.go +++ /dev/null @@ -1,52 +0,0 @@ -package coordinators - -import ( - "context" - "sync" - - "github.com/filecoin-project/lassie/pkg/types" - "go.uber.org/multierr" -) - -func Race(ctx context.Context, queueOperations types.QueueRetrievalsFn) (*types.RetrievalStats, error) { - resultChan := make(chan types.RetrievalResult) - ctx, cancel := context.WithCancel(ctx) - defer cancel() - var waitGroup sync.WaitGroup - waitGroup.Add(1) - go func() { - defer waitGroup.Done() - queueOperations(ctx, func(nextRetrieval types.RetrievalTask) { - waitGroup.Add(1) - go func() { - defer waitGroup.Done() - stats, err := nextRetrieval.Run() - select { - case <-ctx.Done(): - case resultChan <- types.RetrievalResult{Stats: stats, Err: err}: - } - }() - }) - }() - go func() { - waitGroup.Wait() - close(resultChan) - }() - var totalErr error - for { - select { - case result, ok := <-resultChan: - if !ok { - return nil, totalErr - } - if result.Err != nil { - totalErr = multierr.Append(totalErr, result.Err) - } - if result.Stats != nil { - return result.Stats, nil - } - case <-ctx.Done(): - return nil, context.Canceled - } - } -} diff --git a/pkg/retriever/coordinators/race_test.go b/pkg/retriever/coordinators/race_test.go deleted file mode 100644 index 05bbede4..00000000 --- a/pkg/retriever/coordinators/race_test.go +++ /dev/null @@ -1,207 +0,0 @@ -package coordinators_test - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/filecoin-project/go-clock" - "github.com/filecoin-project/lassie/pkg/retriever/coordinators" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/require" - "go.uber.org/multierr" -) - -func TestRace(t *testing.T) { - ctx := context.Background() - testCases := []struct { - name string - results []timeoutResult - getResultsBy time.Duration - contextCancels bool - expectedStats *types.RetrievalStats - expectedErr error - }{ - { - name: "two successes, returns first to finish", - results: []timeoutResult{ - { - duration: 200 * time.Millisecond, - stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - duration: 400 * time.Millisecond, - stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - }, - getResultsBy: 200 * time.Millisecond, - expectedStats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - name: "two errors, returns multi error in sequence", - results: []timeoutResult{ - { - duration: 200 * time.Millisecond, - err: errors.New("something went wrong"), - }, - { - duration: 400 * time.Millisecond, - err: errors.New("something else went wrong"), - }, - }, - getResultsBy: 400 * time.Millisecond, - expectedErr: multierr.Append(errors.New("something went wrong"), errors.New("something else went wrong")), - }, - { - name: "error then success, returns success", - results: []timeoutResult{ - { - duration: 200 * time.Millisecond, - err: errors.New("something went wrong"), - }, - { - duration: 400 * time.Millisecond, - stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - }, - getResultsBy: 400 * time.Millisecond, - expectedStats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - { - name: "success then error, returns success", - results: []timeoutResult{ - { - duration: 200 * time.Millisecond, - stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - duration: 400 * time.Millisecond, - err: errors.New("something went wrong"), - }, - }, - getResultsBy: 200 * time.Millisecond, - expectedStats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - name: "context cancels before any finish, returns context error", - results: []timeoutResult{ - { - duration: 200 * time.Millisecond, - stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - duration: 400 * time.Millisecond, - stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - }, - getResultsBy: 100 * time.Millisecond, - contextCancels: true, - expectedErr: context.Canceled, - }, - { - name: "context cancels after one error, returns context error", - results: []timeoutResult{ - { - duration: 200 * time.Millisecond, - err: errors.New("something went wrong"), - }, - { - duration: 400 * time.Millisecond, - stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - }, - getResultsBy: 300 * time.Millisecond, - contextCancels: true, - expectedErr: context.Canceled, - }, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - clock := clock.NewMock() - startChan := make(chan struct{}) - resultChan := make(chan types.RetrievalResult) - childCtx, childCancel := context.WithCancel(ctx) - defer childCancel() - go func() { - retrievalCalls := func(ctx context.Context, callRetrieval func(types.RetrievalTask)) { - for _, result := range testCase.results { - callRetrieval(types.AsyncRetrievalTask{ - AsyncCandidateRetrieval: &timeoutRetriever{result, ctx, clock, startChan}, - }) - } - } - stats, err := coordinators.Race(childCtx, retrievalCalls) - select { - case <-ctx.Done(): - case resultChan <- types.RetrievalResult{Stats: stats, Err: err}: - } - }() - for range testCase.results { - select { - case <-ctx.Done(): - require.FailNow(t, "failed to start retrievers") - case <-startChan: - } - } - clock.Add(testCase.getResultsBy) - if testCase.contextCancels { - childCancel() - } - select { - case <-ctx.Done(): - require.FailNow(t, "failed to receive result") - case result := <-resultChan: - require.Equal(t, testCase.expectedStats, result.Stats) - require.Equal(t, testCase.expectedErr, result.Err) - } - }) - } -} - -type timeoutResult struct { - duration time.Duration - stats *types.RetrievalStats - err error -} - -type timeoutRetriever struct { - timeoutResult - ctx context.Context - clock clock.Clock - startChan chan<- struct{} -} - -func (t *timeoutRetriever) RetrieveFromAsyncCandidates(types.InboundAsyncCandidates) (*types.RetrievalStats, error) { - timer := t.clock.Timer(t.duration) - t.startChan <- struct{}{} - select { - case <-t.ctx.Done(): - return nil, t.ctx.Err() - case <-timer.C: - return t.stats, t.err - } -} diff --git a/pkg/retriever/coordinators/sequence.go b/pkg/retriever/coordinators/sequence.go deleted file mode 100644 index f0ad7b24..00000000 --- a/pkg/retriever/coordinators/sequence.go +++ /dev/null @@ -1,32 +0,0 @@ -package coordinators - -import ( - "context" - - "github.com/filecoin-project/lassie/pkg/types" - "go.uber.org/multierr" -) - -func Sequence(ctx context.Context, queueOperationsFn types.QueueRetrievalsFn) (*types.RetrievalStats, error) { - var totalErr error - var finalStats *types.RetrievalStats - ctx, cancel := context.WithCancel(ctx) - defer cancel() - queueOperationsFn(ctx, func(retrieval types.RetrievalTask) { - if finalStats != nil { - return - } - stats, err := retrieval.Run() - if err != nil { - totalErr = multierr.Append(totalErr, err) - } - if stats != nil { - finalStats = stats - cancel() - } - }) - if finalStats == nil { - return nil, totalErr - } - return finalStats, nil -} diff --git a/pkg/retriever/coordinators/sequence_test.go b/pkg/retriever/coordinators/sequence_test.go deleted file mode 100644 index 3af32279..00000000 --- a/pkg/retriever/coordinators/sequence_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package coordinators_test - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/filecoin-project/lassie/pkg/retriever/coordinators" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/require" - "go.uber.org/multierr" -) - -func TestSequence(t *testing.T) { - ctx := context.Background() - testCases := []struct { - name string - results []types.RetrievalResult - expectedStats *types.RetrievalStats - expectedErr error - }{ - { - name: "two successes, returns first in sequence", - results: []types.RetrievalResult{ - { - Stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - Stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - }, - expectedStats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - name: "two errors, returns multi error in sequence", - results: []types.RetrievalResult{ - { - Err: errors.New("something went wrong"), - }, - { - Err: errors.New("something else went wrong"), - }, - }, - expectedErr: multierr.Append(errors.New("something went wrong"), errors.New("something else went wrong")), - }, - { - name: "error then success, returns success", - results: []types.RetrievalResult{ - { - Err: errors.New("something went wrong"), - }, - { - Stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - }, - expectedStats: &types.RetrievalStats{ - StorageProviderId: peer.ID("oranges"), - }, - }, - { - name: "success then error, returns success", - results: []types.RetrievalResult{ - { - Stats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - { - Err: errors.New("something went wrong"), - }, - }, - expectedStats: &types.RetrievalStats{ - StorageProviderId: peer.ID("apples"), - }, - }, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - retrievalCalls := func(ctx context.Context, callRetrieval func(types.RetrievalTask)) { - for _, result := range testCase.results { - callRetrieval(types.AsyncRetrievalTask{ - AsyncCandidateRetrieval: &stubRetriever{result}, - }) - } - } - stats, err := coordinators.Sequence(ctx, retrievalCalls) - require.Equal(t, testCase.expectedStats, stats) - require.Equal(t, testCase.expectedErr, err) - }) - } -} - -type stubRetriever struct { - types.RetrievalResult -} - -func (s *stubRetriever) RetrieveFromAsyncCandidates(_ types.InboundAsyncCandidates) (*types.RetrievalStats, error) { - return s.Stats, s.Err -} diff --git a/pkg/retriever/directcandidatesource.go b/pkg/retriever/directcandidatesource.go index 41b72672..4db44619 100644 --- a/pkg/retriever/directcandidatesource.go +++ b/pkg/retriever/directcandidatesource.go @@ -2,236 +2,51 @@ package retriever import ( "context" - "sync" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/lassie/pkg/internal/lp2ptransports" "github.com/filecoin-project/lassie/pkg/types" - // bsnet "github.com/ipfs/boxo/bitswap/network" // DISABLED: bitswap support removed for boxo v0.35.0 compatibility "github.com/ipfs/go-cid" - gsnet "github.com/ipfs/go-graphsync/network" "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" ) var _ types.CandidateSource = &DirectCandidateSource{} // DirectCandidateSource finds candidate protocols from a fixed set of peers type DirectCandidateSource struct { - h host.Host providers []types.Provider } -type Option func(*DirectCandidateSource) - -// WithLibp2pCandidateDiscovery sets a libp2p Host for the DirectCandidateFinder. -// If a Host is set, the providers will be queried to discover available protocols, otherwise -// all protocols will be assumed by default. -func WithLibp2pCandidateDiscovery(h host.Host) Option { - return func(d *DirectCandidateSource) { - d.h = h - } -} - // NewDirectCandidateSource returns a new DirectCandidateFinder for the given providers -func NewDirectCandidateSource(providers []types.Provider, opts ...Option) *DirectCandidateSource { - d := &DirectCandidateSource{ +func NewDirectCandidateSource(providers []types.Provider) *DirectCandidateSource { + return &DirectCandidateSource{ providers: providers, } - for _, opt := range opts { - opt(d) - } - return d -} - -type candidateSender struct { - ctx context.Context - cancel context.CancelFunc - rootCid cid.Cid - candidateResults chan<- types.FindCandidatesResult -} - -func (cs candidateSender) sendCandidate(addr peer.AddrInfo, protocols ...metadata.Protocol) error { - select { - case <-cs.ctx.Done(): - return cs.ctx.Err() - case cs.candidateResults <- types.FindCandidatesResult{Candidate: types.RetrievalCandidate{ - MinerPeer: addr, - RootCid: cs.rootCid, - Metadata: metadata.Default.New(protocols...), - }}: - return nil - } -} - -func (cs candidateSender) sendError(err error) error { - select { - case <-cs.ctx.Done(): - return cs.ctx.Err() - case cs.candidateResults <- types.FindCandidatesResult{Err: err}: - cs.cancel() - return nil - } } -// FindCandidates finds supported protocols for each peer -// TODO: Cache the results? +// FindCandidates returns candidates for each configured provider func (d *DirectCandidateSource) FindCandidates(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error { - candidateResults := make(chan types.FindCandidatesResult) - ctx, cancel := context.WithCancel(ctx) - defer cancel() - cs := candidateSender{ctx, cancel, c, candidateResults} - var wg sync.WaitGroup for _, provider := range d.providers { - - // if protocols are specified, just use those - if len(provider.Protocols) > 0 { - cb(types.RetrievalCandidate{ - MinerPeer: provider.Peer, - RootCid: cs.rootCid, - Metadata: metadata.Default.New(provider.Protocols...), - }) - continue - } - - // if it's http, it'll be in the multiaddr and we can't probe it - if d.trySendHTTPCandidate(provider, c, cb) { - continue - } - - // if we have no libp2p host, just assume all protocols are available - if d.h == nil { - cb(types.RetrievalCandidate{ - MinerPeer: provider.Peer, - RootCid: cs.rootCid, - Metadata: metadata.Default.New(metadata.IpfsGatewayHttp{}, metadata.Bitswap{}, &metadata.GraphsyncFilecoinV1{}), - }) - continue - } - - wg.Add(1) - provider := provider - go func() { - defer wg.Done() - - // probe it - err := d.h.Connect(ctx, provider.Peer) - // don't add peers that we can't connect to - if err != nil { - _ = cs.sendError(err) - return - } - // check for support for Boost libp2p transports protocol - transportsClient := lp2ptransports.NewTransportsClient(d.h) - qr, err := transportsClient.SendQuery(ctx, provider.Peer.ID) - if err == nil { - logger.Debugw("retrieving metadata from transports protocol", "peer", provider.Peer.ID) - // if present, construct metadata from Boost libp2p transports response - d.retrievalCandidatesFromTransportsProtocol(ctx, qr, provider.Peer, cs) - } else { - logger.Debugw("retrieving metadata from libp2p protocol list", "peer", provider.Peer.ID) - // if not present, just make guesses based on list of supported libp2p - // protocols catalogued via identify protocol - d.retrievalCandidatesFromProtocolProbing(ctx, provider.Peer, cs) - } - }() - } - go func() { - wg.Wait() - close(candidateResults) - }() - for { select { case <-ctx.Done(): return ctx.Err() - case result, ok := <-candidateResults: - if !ok { - return nil - } - if result.Err != nil { - return result.Err - } - cb(result.Candidate) + default: } - } -} -func (d *DirectCandidateSource) trySendHTTPCandidate(provider types.Provider, rootCid cid.Cid, cb func(types.RetrievalCandidate)) bool { - if len(provider.Peer.Addrs) != 1 { - return false - } - for _, proto := range provider.Peer.Addrs[0].Protocols() { - if proto.Name == "http" || proto.Name == "https" { + // If protocols are specified, use those + if len(provider.Protocols) > 0 { cb(types.RetrievalCandidate{ MinerPeer: provider.Peer, - RootCid: rootCid, - Metadata: metadata.Default.New(metadata.IpfsGatewayHttp{}), + RootCid: c, + Metadata: metadata.Default.New(provider.Protocols...), }) - return true - } - } - return false -} -func (d *DirectCandidateSource) retrievalCandidatesFromProtocolProbing(ctx context.Context, provider peer.AddrInfo, cs candidateSender) { - var protocols []metadata.Protocol - // DISABLED: bitswap protocol probing removed for boxo v0.35.0 compatibility - // s, err := d.h.NewStream(ctx, provider.ID, - // bsnet.ProtocolBitswap, - // bsnet.ProtocolBitswapOneOne, - // bsnet.ProtocolBitswapOneZero, - // bsnet.ProtocolBitswapNoVers, - // ) - // if err == nil { - // s.Close() - // protocols = append(protocols, &metadata.Bitswap{}) - // } - // must support both graphsync & data transfer to do graphsync filecoin v1 retrieval - s, err := d.h.NewStream(ctx, provider.ID, - gsnet.ProtocolGraphsync_2_0_0) - if err == nil { - s.Close() - s, err = d.h.NewStream(ctx, provider.ID, datatransfer.ProtocolDataTransfer1_2) - if err == nil { - s.Close() - protocols = append(protocols, &metadata.GraphsyncFilecoinV1{}) + continue } - } - _ = cs.sendCandidate(provider, protocols...) -} -func (d *DirectCandidateSource) retrievalCandidatesFromTransportsProtocol(ctx context.Context, qr *lp2ptransports.QueryResponse, provider peer.AddrInfo, cs candidateSender) { - for _, protocol := range qr.Protocols { - // try to parse addr infos directly - addrs, err := peer.AddrInfosFromP2pAddrs(protocol.Addresses...) - // if no peer id is present, use provider's id - if err != nil { - addrs = []peer.AddrInfo{{ - ID: provider.ID, - Addrs: protocol.Addresses, - }} - } - switch protocol.Name { - case "libp2p": - for _, addr := range addrs { - if err := cs.sendCandidate(addr, &metadata.GraphsyncFilecoinV1{}); err != nil { - return - } - } - // DISABLED: bitswap support removed for boxo v0.35.0 compatibility - // case "bitswap": - // for _, addr := range addrs { - // if err := cs.sendCandidate(addr, &metadata.Bitswap{}); err != nil { - // return - // } - // } - case "http": - for _, addr := range addrs { - if err := cs.sendCandidate(addr, &metadata.IpfsGatewayHttp{}); err != nil { - return - } - } - default: - } + // For HTTP-only mode, assume HTTP protocol for all providers + cb(types.RetrievalCandidate{ + MinerPeer: provider.Peer, + RootCid: c, + Metadata: metadata.Default.New(metadata.IpfsGatewayHttp{}), + }) } + return nil } diff --git a/pkg/retriever/directcandidatesource_test.go b/pkg/retriever/directcandidatesource_test.go deleted file mode 100644 index 85acf4d3..00000000 --- a/pkg/retriever/directcandidatesource_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package retriever_test - -import ( - "context" - "testing" - "time" - - "github.com/filecoin-project/lassie/pkg/internal/testutil" - "github.com/filecoin-project/lassie/pkg/retriever" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multiaddr" - "github.com/stretchr/testify/require" -) - -func TestDirectCandidateSourceNoLibp2p(t *testing.T) { - rootCid := testutil.GenerateCid() - p := testutil.GeneratePeers(t, 1)[0] - rawMultiaddr := testutil.GenerateMultiAddr() - httpMultiaddr := testutil.GenerateHTTPMultiAddr() - ctx := context.Background() - testCases := []struct { - name string - provider types.Provider - expectedCandidate types.RetrievalCandidate - }{ - { - name: "peer with protocols", - provider: types.Provider{ - Peer: peer.AddrInfo{ - ID: p, - Addrs: []multiaddr.Multiaddr{rawMultiaddr}, - }, - Protocols: []metadata.Protocol{metadata.IpfsGatewayHttp{}, metadata.Bitswap{}}, - }, - expectedCandidate: types.RetrievalCandidate{ - MinerPeer: peer.AddrInfo{ - ID: p, - Addrs: []multiaddr.Multiaddr{rawMultiaddr}, - }, - RootCid: rootCid, - Metadata: metadata.Default.New(metadata.IpfsGatewayHttp{}, metadata.Bitswap{}), - }, - }, - { - name: "peer with no protocols and standard multiaddr", - provider: types.Provider{ - Peer: peer.AddrInfo{ - ID: p, - Addrs: []multiaddr.Multiaddr{rawMultiaddr}, - }, - }, - expectedCandidate: types.RetrievalCandidate{ - MinerPeer: peer.AddrInfo{ - ID: p, - Addrs: []multiaddr.Multiaddr{rawMultiaddr}, - }, - RootCid: rootCid, - Metadata: metadata.Default.New(metadata.IpfsGatewayHttp{}, metadata.Bitswap{}, &metadata.GraphsyncFilecoinV1{}), - }, - }, - { - name: "peer with no protocols and http multiaddr", - provider: types.Provider{ - Peer: peer.AddrInfo{ - ID: p, - Addrs: []multiaddr.Multiaddr{httpMultiaddr}, - }, - }, - expectedCandidate: types.RetrievalCandidate{ - MinerPeer: peer.AddrInfo{ - ID: p, - Addrs: []multiaddr.Multiaddr{httpMultiaddr}, - }, - RootCid: rootCid, - Metadata: metadata.Default.New(metadata.IpfsGatewayHttp{}), - }, - }, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - d := retriever.NewDirectCandidateSource([]types.Provider{testCase.provider}) - d.FindCandidates(ctx, rootCid, func(candidate types.RetrievalCandidate) { - require.Equal(t, testCase.expectedCandidate, candidate) - }) - }) - } -} diff --git a/pkg/retriever/frontier.go b/pkg/retriever/frontier.go new file mode 100644 index 00000000..7f56d9e0 --- /dev/null +++ b/pkg/retriever/frontier.go @@ -0,0 +1,156 @@ +package retriever + +import ( + "bytes" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/codec/dagjson" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" +) + +// Frontier tracks pending and seen CIDs during streaming DFS traversal. +type Frontier struct { + pending []cid.Cid + seen map[cid.Cid]struct{} +} + +func NewFrontier(root cid.Cid) *Frontier { + return &Frontier{ + pending: []cid.Cid{root}, + seen: make(map[cid.Cid]struct{}), + } +} + +func (f *Frontier) Empty() bool { + return len(f.pending) == 0 +} + +func (f *Frontier) Pop() cid.Cid { + if len(f.pending) == 0 { + return cid.Undef + } + n := len(f.pending) - 1 + c := f.pending[n] + f.pending = f.pending[:n] + return c +} + +func (f *Frontier) Seen(c cid.Cid) bool { + _, ok := f.seen[c] + return ok +} + +func (f *Frontier) MarkSeen(c cid.Cid) { + f.seen[c] = struct{}{} +} + +// PushAll adds CIDs in reverse order so first child is popped first (DFS). +func (f *Frontier) PushAll(cids []cid.Cid) { + for i := len(cids) - 1; i >= 0; i-- { + c := cids[i] + if _, ok := f.seen[c]; !ok { + f.pending = append(f.pending, c) + } + } +} + +func (f *Frontier) Size() int { return len(f.pending) } +func (f *Frontier) SeenCount() int { return len(f.seen) } + +// ExtractLinks returns all CID links from a block based on its codec. +func ExtractLinks(block blocks.Block) ([]cid.Cid, error) { + c := block.Cid() + codec := c.Prefix().Codec + + switch codec { + case cid.Raw: + return nil, nil + case cid.DagProtobuf: + return extractDagPBLinks(block.RawData()) + case cid.DagCBOR: + return extractDagCBORLinks(block.RawData()) + case cid.DagJSON: + return extractDagJSONLinks(block.RawData()) + default: + // unknown codec — try CBOR, fall back to no links + links, err := extractDagCBORLinks(block.RawData()) + if err != nil { + return nil, nil + } + return links, nil + } +} + +func extractDagPBLinks(data []byte) ([]cid.Cid, error) { + nb := dagpb.Type.PBNode.NewBuilder() + if err := dagpb.DecodeBytes(nb, data); err != nil { + return nil, err + } + node := nb.Build().(dagpb.PBNode) + + var cids []cid.Cid + iter := node.Links.ListIterator() + for !iter.Done() { + _, linkNode, err := iter.Next() + if err != nil { + return nil, err + } + link := linkNode.(dagpb.PBLink) + cids = append(cids, link.Hash.Link().(cidlink.Link).Cid) + } + return cids, nil +} + +func extractDagCBORLinks(data []byte) ([]cid.Cid, error) { + nb := basicnode.Prototype.Any.NewBuilder() + if err := dagcbor.Decode(nb, bytes.NewReader(data)); err != nil { + return nil, err + } + return collectLinks(nb.Build()), nil +} + +func extractDagJSONLinks(data []byte) ([]cid.Cid, error) { + nb := basicnode.Prototype.Any.NewBuilder() + if err := dagjson.Decode(nb, bytes.NewReader(data)); err != nil { + return nil, err + } + return collectLinks(nb.Build()), nil +} + +func collectLinks(node ipld.Node) []cid.Cid { + var links []cid.Cid + + switch node.Kind() { + case ipld.Kind_Link: + if link, err := node.AsLink(); err == nil { + if cl, ok := link.(cidlink.Link); ok { + links = append(links, cl.Cid) + } + } + case ipld.Kind_Map: + iter := node.MapIterator() + for !iter.Done() { + _, v, err := iter.Next() + if err != nil { + break + } + links = append(links, collectLinks(v)...) + } + case ipld.Kind_List: + iter := node.ListIterator() + for !iter.Done() { + _, v, err := iter.Next() + if err != nil { + break + } + links = append(links, collectLinks(v)...) + } + } + + return links +} diff --git a/pkg/retriever/frontier_test.go b/pkg/retriever/frontier_test.go new file mode 100644 index 00000000..d999ec04 --- /dev/null +++ b/pkg/retriever/frontier_test.go @@ -0,0 +1,57 @@ +package retriever + +import ( + "testing" + + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +func TestFrontier(t *testing.T) { + c1 := makeCid("block1") + c2 := makeCid("block2") + c3 := makeCid("block3") + + f := NewFrontier(c1) + + // Initially has root + require.False(t, f.Empty()) + require.Equal(t, 1, f.Size()) + require.Equal(t, 0, f.SeenCount()) + + // Pop returns root + got := f.Pop() + require.Equal(t, c1, got) + require.True(t, f.Empty()) + + // Mark as seen + f.MarkSeen(c1) + require.True(t, f.Seen(c1)) + require.False(t, f.Seen(c2)) + require.Equal(t, 1, f.SeenCount()) + + // PushAll adds children (in reverse for DFS) + f.PushAll([]cid.Cid{c2, c3}) + require.Equal(t, 2, f.Size()) + + // DFS order: c2 first (was added last due to reverse) + got = f.Pop() + require.Equal(t, c2, got) + + got = f.Pop() + require.Equal(t, c3, got) + require.True(t, f.Empty()) + + // PushAll skips seen CIDs + f.MarkSeen(c2) + f.PushAll([]cid.Cid{c1, c2, c3}) // c1 already seen + require.Equal(t, 1, f.Size()) // Only c3 added + got = f.Pop() + require.Equal(t, c3, got) +} + +func makeCid(s string) cid.Cid { + h, _ := mh.Sum([]byte(s), mh.SHA2_256, -1) + return cid.NewCidV1(cid.Raw, h) +} diff --git a/pkg/retriever/graphsyncretriever.go b/pkg/retriever/graphsyncretriever.go deleted file mode 100644 index 08bfd596..00000000 --- a/pkg/retriever/graphsyncretriever.go +++ /dev/null @@ -1,247 +0,0 @@ -package retriever - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/filecoin-project/go-clock" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - retrievaltypes "github.com/filecoin-project/go-retrieval-types" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lassie/pkg/events" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/dagjson" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" - "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multicodec" -) - -// Connect() may be a near-noop for already-connect libp2p connections, so this -// allows parallel goroutines of already-connected peers to queue and have the -// scoring logic to select one to start. -const GraphsyncDefaultInitialWait = 2 * time.Millisecond - -type GraphsyncClient interface { - Connect(ctx context.Context, peerAddr peer.AddrInfo) error - RetrieveFromPeer( - ctx context.Context, - linkSystem ipld.LinkSystem, - peerID peer.ID, - proposal *retrievaltypes.DealProposal, - selector ipld.Node, - maxLinks uint64, - eventsCallback datatransfer.Subscriber, - gracefulShutdownRequested <-chan struct{}, - ) (*types.RetrievalStats, error) -} - -var _ TransportProtocol = &ProtocolGraphsync{} - -type ProtocolGraphsync struct { - Client GraphsyncClient - Clock clock.Clock -} - -// NewGraphsyncRetriever makes a new CandidateRetriever for Graphsync retrievals -// (transport-graphsync-filecoinv1). -func NewGraphsyncRetriever(session Session, client GraphsyncClient) types.CandidateRetriever { - return NewGraphsyncRetrieverWithConfig(session, client, clock.New(), GraphsyncDefaultInitialWait, false) -} - -func NewGraphsyncRetrieverWithConfig( - session Session, - client GraphsyncClient, - clock clock.Clock, - initialPause time.Duration, - noDirtyClose bool, -) types.CandidateRetriever { - return ¶llelPeerRetriever{ - Protocol: &ProtocolGraphsync{ - Client: client, - Clock: clock, - }, - Session: session, - Clock: clock, - QueueInitialPause: initialPause, - noDirtyClose: noDirtyClose, - } -} - -func (pg ProtocolGraphsync) Code() multicodec.Code { - return multicodec.TransportGraphsyncFilecoinv1 -} - -func (pg ProtocolGraphsync) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol { - gsNewMetadata, ok := newMetadata.(*metadata.GraphsyncFilecoinV1) - // Normally we should only get full GraphsyncFilecoinV1 metadata, but not - // if the candidate didn't come from the indexer. Since we depend on the - // metadata for comparison, we need to make sure we have some. - if !ok { - gsNewMetadata = &metadata.GraphsyncFilecoinV1{PieceCID: cid} - } - if currentMetadata != nil { // seen this candidate before - if !ok { - return currentMetadata - } - gsCurrentMetadata := currentMetadata.(*metadata.GraphsyncFilecoinV1) - if !graphsyncMetadataCompare(gsNewMetadata, gsCurrentMetadata, false) { - return currentMetadata // old one is better - } - } - return gsNewMetadata -} - -// graphsyncMetadataCompare compares two metadata.GraphsyncFilecoinV1s and -// returns true if the first is preferable to the second. -// NOTE this is similar to comparisons used in Session#CompareCandidates -func graphsyncMetadataCompare(a, b *metadata.GraphsyncFilecoinV1, defaultValue bool) bool { - // prioritize verified deals over not verified deals - if a.VerifiedDeal != b.VerifiedDeal { - return a.VerifiedDeal - } - - // prioritize fast retrievel over not fast retrieval - if a.FastRetrieval != b.FastRetrieval { - return a.FastRetrieval - } - - return defaultValue -} - -func (pg *ProtocolGraphsync) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error) { - startTime := pg.Clock.Now() - if err := pg.Client.Connect(ctx, candidate.MinerPeer); err != nil { - return 0, err - } - return pg.Clock.Since(startTime), nil -} - -func (pg *ProtocolGraphsync) Retrieve( - ctx context.Context, - retrieval *retrieval, - shared *retrievalShared, - timeout time.Duration, - candidate types.RetrievalCandidate, -) (*types.RetrievalStats, error) { - retrievalStart := pg.Clock.Now() - - ss := "*" - selector := retrieval.request.GetSelector() - if !ipld.DeepEqual(selector, selectorparse.CommonSelector_ExploreAllRecursively) { - byts, err := ipld.Encode(selector, dagjson.Encode) - if err != nil { - return nil, err - } - ss = string(byts) - } - - logger.Infof( - "Attempting retrieval from SP %s for %s (with selector: [%s])", - candidate.MinerPeer.ID, - candidate.RootCid, - ss, - ) - - params, err := retrievaltypes.NewParamsV1(big.Zero(), 0, 0, selector, nil, big.Zero()) - if err != nil { - return nil, fmt.Errorf("%w; %w; %w", ErrRetrievalFailed, ErrProposalCreationFailed, err) - } - proposal := &retrievaltypes.DealProposal{ - PayloadCID: candidate.RootCid, - ID: retrievaltypes.DealID(dealIdGen.Next()), - Params: params, - } - - retrieveCtx, retrieveCancel := context.WithCancel(ctx) - defer retrieveCancel() - - var lastBytesReceived uint64 - var doneLk sync.Mutex - var done, timedOut bool - var lastBytesReceivedTimer, gracefulShutdownTimer *clock.Timer - - gracefulShutdownChan := make(chan struct{}, 1) - - // Start the timeout tracker only if retrieval timeout isn't 0 - if timeout != 0 { - lastBytesReceivedTimer = retrieval.parallelPeerRetriever.Clock.AfterFunc(timeout, func() { - doneLk.Lock() - done = true - timedOut = true - doneLk.Unlock() - - gracefulShutdownChan <- struct{}{} - gracefulShutdownTimer = retrieval.parallelPeerRetriever.Clock.AfterFunc(1*time.Minute, retrieveCancel) - }) - } - - var receivedFirstByte bool - var totalReceived uint64 - eventsSubscriber := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - switch event.Code { - case datatransfer.Open: - shared.sendEvent(ctx, events.Proposed(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate)) - case datatransfer.NewVoucherResult: - lastVoucher := channelState.LastVoucherResult() - resType, err := retrievaltypes.DealResponseFromNode(lastVoucher.Voucher) - if err != nil { - return - } - if resType.Status == retrievaltypes.DealStatusAccepted { - shared.sendEvent(ctx, events.Accepted(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate)) - } - case datatransfer.DataReceivedProgress: - if !receivedFirstByte { - receivedFirstByte = true - shared.sendEvent(ctx, events.FirstByte(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Clock.Since(retrievalStart), multicodec.TransportGraphsyncFilecoinv1)) - } - lastReceived := totalReceived - totalReceived = channelState.Received() - if totalReceived > lastReceived { - shared.sendEvent(ctx, events.BlockReceived(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate, multicodec.TransportGraphsyncFilecoinv1, totalReceived-lastReceived)) - } - if lastBytesReceivedTimer != nil { - doneLk.Lock() - if !done { - if lastBytesReceived != channelState.Received() { - lastBytesReceivedTimer.Reset(timeout) - lastBytesReceived = channelState.Received() - } - } - doneLk.Unlock() - } - } - } - - stats, err := pg.Client.RetrieveFromPeer( - retrieveCtx, - retrieval.request.LinkSystem, - candidate.MinerPeer.ID, - proposal, - selector, - uint64(retrieval.request.MaxBlocks), - eventsSubscriber, - gracefulShutdownChan, - ) - - if timedOut { - return nil, fmt.Errorf("%w; %w after %s", ErrRetrievalFailed, ErrRetrievalTimedOut, timeout) - } - - if lastBytesReceivedTimer != nil { - lastBytesReceivedTimer.Stop() - } - if gracefulShutdownTimer != nil { - gracefulShutdownTimer.Stop() - } - - if err != nil { - return nil, fmt.Errorf("%w; %w", ErrRetrievalFailed, err) - } - return stats, nil -} diff --git a/pkg/retriever/graphsyncretriever_test.go b/pkg/retriever/graphsyncretriever_test.go deleted file mode 100644 index 061b3944..00000000 --- a/pkg/retriever/graphsyncretriever_test.go +++ /dev/null @@ -1,1115 +0,0 @@ -package retriever_test - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/filecoin-project/go-clock" - "github.com/filecoin-project/lassie/pkg/events" - "github.com/filecoin-project/lassie/pkg/internal/testutil" - "github.com/filecoin-project/lassie/pkg/retriever" - "github.com/filecoin-project/lassie/pkg/session" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/google/uuid" - "github.com/ipfs/go-cid" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" - trustlessutils "github.com/ipld/go-trustless-utils" - "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multicodec" - "github.com/stretchr/testify/require" -) - -func TestRetrievalRacing(t *testing.T) { - retrievalID := types.RetrievalID(uuid.New()) - startTime := time.Now().Add(time.Hour) - initialPause := 10 * time.Millisecond - peerFoo := peer.ID("foo") - peerBar := peer.ID("bar") - peerBaz := peer.ID("baz") - peerBang := peer.ID("bang") - - testCases := []struct { - name string - customMetadata map[string]metadata.Protocol - connectReturns map[string]testutil.DelayedConnectReturn - retrievalReturns map[string]testutil.DelayedClientReturn - cancelAfter time.Duration - expectedRetrieval string - expectSequence []testutil.ExpectedActionsAtTime - }{ - { - name: "single fast", - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Delay: time.Millisecond * 20}, - "bar": {Delay: time.Millisecond * 500}, - "baz": {Delay: time.Millisecond * 500}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultStats: &types.RetrievalStats{StorageProviderId: peerFoo, Size: 1}, Delay: time.Millisecond * 20}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: time.Millisecond * 500}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 3}, Delay: time.Millisecond * 500}, - }, - expectedRetrieval: "foo", - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 20, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 20}, - }, - }, - { - AfterStart: time.Millisecond*20 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond*40 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.Accepted(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FirstByte(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, time.Millisecond*20, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, 1, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerFoo, Duration: time.Millisecond * 20}, - {Type: testutil.SessionMetric_Success, Provider: peerFoo, Value: 1.0}, - }, - }, - }, - }, - { - name: "all connects finished", - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Delay: time.Millisecond * 20}, - "bar": {Delay: time.Millisecond * 50}, - "baz": {Delay: time.Millisecond * 50}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultStats: &types.RetrievalStats{StorageProviderId: peerFoo, Size: 1}, Delay: time.Millisecond * 500}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: time.Millisecond * 500}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 3}, Delay: time.Millisecond * 500}, - }, - expectedRetrieval: "foo", - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 20, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 20}, - }, - }, - { - AfterStart: time.Millisecond*20 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond*50 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.ConnectedToProvider(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 50}, - {Type: testutil.SessionMetric_Connect, Provider: peerBaz, Duration: time.Millisecond * 50}, - }, - }, - { - AfterStart: time.Millisecond*520 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*520+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.Accepted(startTime.Add(time.Millisecond*520+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FirstByte(startTime.Add(time.Millisecond*520+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, time.Millisecond*500, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*520+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*520+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, 1, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerFoo, Duration: time.Millisecond * 500}, - {Type: testutil.SessionMetric_Success, Provider: peerFoo, Value: 1.0}, - }, - }, - }, - }, - { - name: "all connects failed", - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Err: errors.New("Nope"), Delay: time.Millisecond * 20}, - "bar": {Err: errors.New("Nope"), Delay: time.Millisecond * 20}, - "baz": {Err: errors.New("Nope"), Delay: time.Millisecond * 20}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultStats: &types.RetrievalStats{StorageProviderId: peerFoo, Size: 1}, Delay: time.Millisecond}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: time.Millisecond}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 3}, Delay: time.Millisecond}, - }, - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 20, - ExpectedEvents: []types.RetrievalEvent{ - events.FailedRetrieval(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, "unable to connect to provider: Nope"), - events.FailedRetrieval(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1, "unable to connect to provider: Nope"), - events.FailedRetrieval(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1, "unable to connect to provider: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerFoo}, - {Type: testutil.SessionMetric_Failure, Provider: peerBar}, - {Type: testutil.SessionMetric_Failure, Provider: peerBaz}, - }, - }, - }, - }, - { - name: "first retrieval failed", - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Err: nil, Delay: time.Millisecond * 20}, - "bar": {Err: nil, Delay: time.Millisecond * 60}, - "baz": {Err: nil, Delay: time.Millisecond * 500}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultErr: errors.New("Nope"), Delay: time.Millisecond * 20}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: time.Millisecond * 20}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 3}, Delay: time.Millisecond * 20}, - }, - expectedRetrieval: "bar", - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 20, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 20}, - }, - }, - { - AfterStart: time.Millisecond*20 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond*40 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerFoo}, - }, - }, - { - AfterStart: time.Millisecond * 60, - ReceivedRetrievals: []peer.ID{peerBar}, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*60), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 60}, - }, - }, - { - AfterStart: time.Millisecond * 80, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*80), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}), - events.Accepted(startTime.Add(time.Millisecond*80), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}), - events.FirstByte(startTime.Add(time.Millisecond*80), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, time.Millisecond*20, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*80), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*80), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, 2, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerBar, Duration: time.Millisecond * 20}, - {Type: testutil.SessionMetric_Success, Provider: peerBar, Value: 2}, - }, - }, - }, - }, - - { - name: "all retrievals failed", - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Err: nil, Delay: time.Millisecond * 20}, - "bar": {Err: nil, Delay: time.Millisecond * 25}, - "baz": {Err: nil, Delay: time.Millisecond * 60}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultErr: errors.New("Nope"), Delay: time.Millisecond * 100}, - "bar": {ResultErr: errors.New("Nope"), Delay: time.Millisecond * 100}, - "baz": {ResultErr: errors.New("Nope"), Delay: time.Millisecond * 100}, - }, - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 20, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 20}, - }, - }, - { - AfterStart: time.Millisecond * 25, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*25), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 25}, - }, - }, - { - AfterStart: time.Millisecond*20 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond * 60, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*60), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBaz, Duration: time.Millisecond * 60}, - }, - }, - { - AfterStart: time.Millisecond*120 + initialPause, - ReceivedRetrievals: []peer.ID{peerBar}, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*120+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*120+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerFoo}, - }, - }, - { - AfterStart: time.Millisecond*220 + initialPause, - ReceivedRetrievals: []peer.ID{peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*220+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*220+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerBar}, - }, - }, - { - AfterStart: time.Millisecond*320 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*320+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*320+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerBaz}, - }, - }, - }, - }, - - // quickest ("foo") fails retrieval, the other 3 line up in the queue, - // it should choose the "best", which is the one that's fast retrieval/verified deal - { - name: "racing chooses best", - customMetadata: map[string]metadata.Protocol{ - "bar": &metadata.GraphsyncFilecoinV1{FastRetrieval: true, VerifiedDeal: false}, - "bang": &metadata.GraphsyncFilecoinV1{FastRetrieval: false, VerifiedDeal: true}, - }, - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Err: nil, Delay: time.Millisecond}, - "bar": {Err: nil, Delay: time.Millisecond * 100}, - "baz": {Err: nil, Delay: time.Millisecond * 100}, - "bang": {Err: nil, Delay: time.Millisecond * 100}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultErr: errors.New("Nope"), Delay: time.Millisecond * 200}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 3}, Delay: time.Millisecond * 20}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 2}, Delay: time.Millisecond * 20}, - "bang": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBang, Size: 4}, Delay: time.Millisecond * 20}, - }, - expectedRetrieval: "baz", - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz, peerBang}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 1, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*1), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 1}, - }, - }, - { - AfterStart: time.Millisecond*1 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond * 100, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 100}, - {Type: testutil.SessionMetric_Connect, Provider: peerBaz, Duration: time.Millisecond * 100}, - {Type: testutil.SessionMetric_Connect, Provider: peerBang, Duration: time.Millisecond * 100}, - }, - }, - { - AfterStart: time.Millisecond*201 + initialPause, - ReceivedRetrievals: []peer.ID{peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*201+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*201+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerFoo}, - }, - }, - { - AfterStart: time.Millisecond*221 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*221+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}), - events.Accepted(startTime.Add(time.Millisecond*221+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}), - events.FirstByte(startTime.Add(time.Millisecond*221+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, time.Millisecond*20, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*221+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*221+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, 2, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerBaz, Duration: time.Millisecond * 20}, - {Type: testutil.SessionMetric_Success, Provider: peerBaz, Value: 2}, - }, - }, - }, - }, - - // same as above, but we don't have a failing "foo" to act as a gate for the - // others to line up; tests the prioritywaitqueue initial pause - { - name: "racing chooses best (same connect)", - customMetadata: map[string]metadata.Protocol{ - "bar": &metadata.GraphsyncFilecoinV1{FastRetrieval: true, VerifiedDeal: false}, - "bang": &metadata.GraphsyncFilecoinV1{FastRetrieval: false, VerifiedDeal: true}, - }, - connectReturns: map[string]testutil.DelayedConnectReturn{ - "bar": {Err: nil, Delay: time.Millisecond * 100}, - "baz": {Err: nil, Delay: time.Millisecond * 100}, - "bang": {Err: nil, Delay: time.Millisecond * 100}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 3}, Delay: time.Millisecond * 20}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 2}, Delay: time.Millisecond * 20}, - "bang": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBang, Size: 4}, Delay: time.Millisecond * 20}, - }, - expectedRetrieval: "baz", - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerBar, peerBaz, peerBang}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 100, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 100}, - {Type: testutil.SessionMetric_Connect, Provider: peerBaz, Duration: time.Millisecond * 100}, - {Type: testutil.SessionMetric_Connect, Provider: peerBang, Duration: time.Millisecond * 100}, - }, - }, - { - AfterStart: time.Millisecond*100 + initialPause, - ReceivedRetrievals: []peer.ID{peerBaz}, - }, - { - AfterStart: time.Millisecond*120 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*120+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}), - events.Accepted(startTime.Add(time.Millisecond*120+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}), - events.FirstByte(startTime.Add(time.Millisecond*120+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, time.Millisecond*20, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*120+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*120+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, 2, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerBaz, Duration: time.Millisecond * 20}, - {Type: testutil.SessionMetric_Success, Provider: peerBaz, Value: 2}, - }, - }, - }, - }, - - // quickest ("foo") fails retrieval, the other 3 line up in the queue, - // it should choose the "best", which in this case is the fastest to line - // up / connect (they are all free and the same size) - { - name: "racing chooses fastest connect", - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Err: nil, Delay: time.Millisecond * 1}, - "bar": {Err: nil, Delay: time.Millisecond * 220}, - "baz": {Err: nil, Delay: time.Millisecond * 200}, - "bang": {Err: nil, Delay: time.Millisecond * 100}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultErr: errors.New("Nope"), Delay: time.Millisecond * 400}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 3}, Delay: time.Millisecond * 20}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 2}, Delay: time.Millisecond * 20}, - "bang": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBang, Size: 4}, Delay: time.Millisecond * 20}, - }, - expectedRetrieval: "bang", - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz, peerBang}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 1, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*1), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 1}, - }, - }, - { - AfterStart: time.Millisecond*1 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond * 100, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBang, Duration: time.Millisecond * 100}, - }, - }, - { - AfterStart: time.Millisecond * 200, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*200), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBaz, Duration: time.Millisecond * 200}, - }, - }, - { - AfterStart: time.Millisecond * 220, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*220), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 220}, - }, - }, - { - AfterStart: time.Millisecond*401 + initialPause, - ReceivedRetrievals: []peer.ID{peerBang}, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*401+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*401+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerFoo}, - }, - }, - { - AfterStart: time.Millisecond*421 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*421+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}), - events.Accepted(startTime.Add(time.Millisecond*421+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}), - events.FirstByte(startTime.Add(time.Millisecond*421+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, time.Millisecond*20, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*421+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*421+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, 4, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerBang, Duration: time.Millisecond * 20}, - {Type: testutil.SessionMetric_Success, Provider: peerBang, Value: 4}, - }, - }, - }, - }, - { - // Unlike the test with the same name against the full Retriever, - // this test operates directly on the GraphsyncRetriever, so we can - // tell it `noDirtyClose=true`, cancel the context, and check - // whether it's still running or not. Proper completion of this test - // indicates proper cleanup of per-SP goroutines. - name: "context cancelled before completed", - connectReturns: map[string]testutil.DelayedConnectReturn{ - "foo": {Delay: time.Millisecond * 20}, - "bar": {Delay: time.Millisecond * 50}, - "baz": {Delay: time.Millisecond * 50}, - }, - retrievalReturns: map[string]testutil.DelayedClientReturn{ - "foo": {ResultStats: &types.RetrievalStats{StorageProviderId: peerFoo, Size: 1}, Delay: time.Second * 1}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: time.Second * 2}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 3}, Delay: time.Second * 3}, - }, - cancelAfter: time.Millisecond * 100, - expectSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 20, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*20), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 20}, - }, - }, - { - AfterStart: time.Millisecond*50 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.ConnectedToProvider(startTime.Add(time.Millisecond*40+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 50}, - {Type: testutil.SessionMetric_Connect, Provider: peerBaz, Duration: time.Millisecond * 50}, - }, - }, - { - AfterStart: time.Millisecond*20 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond * 100, - // cancellation here - }, - { - AfterStart: time.Millisecond * 500, - // cancellation here - }, - }, - }, - } - ctx := context.Background() - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - clock := clock.NewMock() - clock.Set(startTime) - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - retCtx, retCancel := context.WithCancel(ctx) - defer retCancel() - mockClient := testutil.NewMockClient(tc.connectReturns, tc.retrievalReturns, clock) - candidates := []types.RetrievalCandidate{} - for p := range tc.connectReturns { - var protocol metadata.Protocol - if custom, ok := tc.customMetadata[p]; ok { - protocol = custom - } else { - protocol = &metadata.GraphsyncFilecoinV1{VerifiedDeal: true, FastRetrieval: true} - } - candidates = append(candidates, types.NewRetrievalCandidate(peer.ID(p), nil, cid.Undef, protocol)) - } - - // we're testing the actual Session implementation, but we also want - // to observe, so MockSession WithActual lets us do that. - scfg := session.DefaultConfig(). - WithDefaultProviderConfig(session.ProviderConfig{ - RetrievalTimeout: time.Second, - }). - WithConnectTimeAlpha(0.0). // only use the last connect time - WithoutRandomness() - _session := session.NewSession(scfg, true) - session := testutil.NewMockSession(ctx) - session.WithActual(_session) - // register the retrieval so we don't get any "not found" errors out - // of the session - session.RegisterRetrieval(retrievalID, cid.Undef, basicnode.NewString("r")) - session.AddToRetrieval(retrievalID, []peer.ID{peerFoo, peerBar, peerBaz, peerBang}) - - cfg := retriever.NewGraphsyncRetrieverWithConfig(session, mockClient, clock, initialPause, true) - - rv := testutil.RetrievalVerifier{ - ExpectedSequence: tc.expectSequence, - } - // perform retrieval and make sure we got a result - results := rv.RunWithVerification( - ctx, - t, - clock, - mockClient, - nil, - session, - retCancel, - tc.cancelAfter, - []testutil.RunRetrieval{func(cb func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - return cfg.Retrieve(retCtx, types.RetrievalRequest{ - Request: trustlessutils.Request{Root: cid.Undef}, - RetrievalID: retrievalID, - LinkSystem: cidlink.DefaultLinkSystem(), - }, cb).RetrieveFromAsyncCandidates(makeAsyncCandidates(t, candidates)) - }}) - require.Len(t, results, 1) - stats, err := results[0].Stats, results[0].Err - if tc.expectedRetrieval != "" { - require.NotNil(t, stats) - require.NoError(t, err) - // make sure we got the final retrieval we wanted - require.Equal(t, tc.retrievalReturns[tc.expectedRetrieval].ResultStats, stats) - } else { - require.Nil(t, stats) - require.Error(t, err) - } - }) - } -} - -// run two retrievals simultaneously -func TestMultipleRetrievals(t *testing.T) { - retrievalID1 := types.RetrievalID(uuid.New()) - retrievalID2 := types.RetrievalID(uuid.New()) - peerFoo := peer.ID("foo") - peerBar := peer.ID("bar") - peerBaz := peer.ID("baz") - peerBang := peer.ID("bang") - peerBoom := peer.ID("boom") - peerBing := peer.ID("bing") - cid1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") - cid2 := cid.MustParse("bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk") - startTime := time.Now().Add(time.Hour) - clock := clock.NewMock() - initialPause := 10 * time.Millisecond - clock.Set(startTime) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - mockClient := testutil.NewMockClient( - map[string]testutil.DelayedConnectReturn{ - // group a - "foo": {Err: nil, Delay: time.Millisecond}, - "bar": {Err: nil, Delay: time.Millisecond * 100}, - "baz": {Err: nil, Delay: time.Millisecond * 500}, // should not finish this - // group b - "bang": {Err: nil, Delay: time.Millisecond * 500}, // should not finish this - "boom": {Err: errors.New("Nope"), Delay: time.Millisecond}, - "bing": {Err: nil, Delay: time.Millisecond * 100}, - }, - map[string]testutil.DelayedClientReturn{ - // group a - "foo": {ResultErr: errors.New("Nope"), Delay: time.Millisecond}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: time.Millisecond * 200}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 3}, Delay: time.Millisecond * 200}, - // group b - "bang": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBang, Size: 3}, Delay: time.Millisecond * 201}, - "boom": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBoom, Size: 3}, Delay: time.Millisecond * 201}, - "bing": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBing, Size: 3}, Delay: time.Millisecond * 201}, - }, - clock, - ) - - expectedSequence := []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz, peerBang, peerBoom, peerBing}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBang}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBoom}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBing}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 1, - ExpectedEvents: []types.RetrievalEvent{ - events.FailedRetrieval(startTime.Add(time.Millisecond*1), retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBoom}}, multicodec.TransportGraphsyncFilecoinv1, "unable to connect to provider: Nope"), - events.ConnectedToProvider(startTime.Add(time.Millisecond*1), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerBoom}, - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond}, - }, - }, - { - AfterStart: time.Millisecond*1 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond*2 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*2+initialPause), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*2+initialPause), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerFoo}, - }, - }, - { - AfterStart: time.Millisecond * 100, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBing}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 100}, - {Type: testutil.SessionMetric_Connect, Provider: peerBing, Duration: time.Millisecond * 100}, - }, - }, - { - AfterStart: time.Millisecond * 100, - ReceivedRetrievals: []peer.ID{peerBar}, - }, - { - AfterStart: time.Millisecond*100 + initialPause, - ReceivedRetrievals: []peer.ID{peerBing}, - }, - { - AfterStart: time.Millisecond * 300, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*300), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}), - events.Accepted(startTime.Add(time.Millisecond*300), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}), - events.FirstByte(startTime.Add(time.Millisecond*300), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, time.Millisecond*200, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*300), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*300), retrievalID1, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, 2, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerBar, Duration: time.Millisecond * 200}, - {Type: testutil.SessionMetric_Success, Provider: peerBar, Value: 2}, - }, - }, - { - AfterStart: time.Millisecond*301 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*301+initialPause), retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBing}}), - events.Accepted(startTime.Add(time.Millisecond*301+initialPause), retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBing}}), - events.FirstByte(startTime.Add(time.Millisecond*301+initialPause), retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBing}}, time.Millisecond*201, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*301+initialPause), retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBing}}, multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*301+initialPause), retrievalID2, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBing}}, 3, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerBing, Duration: time.Millisecond * 201}, - {Type: testutil.SessionMetric_Success, Provider: peerBing, Value: 3}, - }, - }, - } - - // we're testing the actual Session implementation, but we also want - // to observe, so MockSession WithActual lets us do that. - scfg := session.DefaultConfig(). - WithDefaultProviderConfig(session.ProviderConfig{ - RetrievalTimeout: time.Second, - }). - WithConnectTimeAlpha(0.0). // only use the last connect time - WithoutRandomness() - _session := session.NewSession(scfg, true) - session := testutil.NewMockSession(ctx) - session.WithActual(_session) - // register the retrieval so we don't get any "not found" errors out - // of the session - session.RegisterRetrieval(retrievalID1, cid.MustParse("bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk"), basicnode.NewString("r")) - session.AddToRetrieval(retrievalID1, []peer.ID{peerFoo, peerBar, peerBaz}) - session.RegisterRetrieval(retrievalID2, cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), basicnode.NewString("r")) - session.AddToRetrieval(retrievalID2, []peer.ID{peerBang, peerBoom, peerBing}) - - cfg := retriever.NewGraphsyncRetrieverWithConfig(session, mockClient, clock, initialPause, true) - - results := testutil.RetrievalVerifier{ - ExpectedSequence: expectedSequence, - }.RunWithVerification(ctx, t, clock, mockClient, nil, session, nil, 0, []testutil.RunRetrieval{ - func(cb func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - return cfg.Retrieve(context.Background(), types.RetrievalRequest{ - Request: trustlessutils.Request{Root: cid1}, - RetrievalID: retrievalID1, - LinkSystem: cidlink.DefaultLinkSystem(), - }, cb).RetrieveFromAsyncCandidates(makeAsyncCandidates(t, []types.RetrievalCandidate{ - types.NewRetrievalCandidate(peerFoo, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{}), - types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{}), - types.NewRetrievalCandidate(peerBaz, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{}), - })) - }, - func(cb func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - return cfg.Retrieve(context.Background(), types.RetrievalRequest{ - Request: trustlessutils.Request{Root: cid2}, - RetrievalID: retrievalID2, - LinkSystem: cidlink.DefaultLinkSystem(), - }, cb).RetrieveFromAsyncCandidates(makeAsyncCandidates(t, []types.RetrievalCandidate{ - types.NewRetrievalCandidate(peerBang, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{}), - types.NewRetrievalCandidate(peerBoom, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{}), - types.NewRetrievalCandidate(peerBing, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{}), - })) - }}) - require.Len(t, results, 2) - stats, err := results[0].Stats, results[0].Err - require.NoError(t, err) - require.NotNil(t, stats) - // make sure we got the final retrieval we wanted - require.Equal(t, mockClient.GetRetrievalReturns()["bar"].ResultStats, stats) - - stats, err = results[1].Stats, results[1].Err - require.NoError(t, err) - require.NotNil(t, stats) - // make sure we got the final retrieval we wanted - require.Equal(t, mockClient.GetRetrievalReturns()["bing"].ResultStats, stats) -} - -func TestRetrievalSelector(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - retrievalID := types.RetrievalID(uuid.New()) - peerFoo := peer.ID("foo") - peerBar := peer.ID("bar") - cid1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") - mockClient := testutil.NewMockClient( - map[string]testutil.DelayedConnectReturn{"foo": {Err: nil, Delay: 0}}, - map[string]testutil.DelayedClientReturn{"foo": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: 0}}, - clock.New(), - ) - - scfg := session.DefaultConfig(). - WithDefaultProviderConfig(session.ProviderConfig{ - RetrievalTimeout: time.Second, - }). - WithConnectTimeAlpha(0.0). // only use the last connect time - WithoutRandomness() - session := session.NewSession(scfg, true) - cfg := retriever.NewGraphsyncRetriever(session, mockClient) - - selector := selectorparse.CommonSelector_MatchPoint - - retrieval := cfg.Retrieve(context.Background(), types.RetrievalRequest{ - Request: trustlessutils.Request{Root: cid1}, - RetrievalID: retrievalID, - LinkSystem: cidlink.DefaultLinkSystem(), - Selector: selector, - }, nil) - stats, err := retrieval.RetrieveFromAsyncCandidates(makeAsyncCandidates(t, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerFoo, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{})})) - require.NoError(t, err) - require.NotNil(t, stats) - require.Equal(t, mockClient.GetRetrievalReturns()["foo"].ResultStats, stats) - - // make sure we performed the retrievals we expected - rr := mockClient.VerifyReceivedRetrievalFrom(ctx, t, peerFoo) - require.NotNil(t, rr) - require.Same(t, selector, rr.Selector) -} - -func TestDuplicateRetreivals(t *testing.T) { - retrievalID := types.RetrievalID(uuid.New()) - peerFoo := peer.ID("foo") - peerBar := peer.ID("bar") - peerBaz := peer.ID("baz") - cid1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") - startTime := time.Now().Add(time.Hour) - clock := clock.NewMock() - clock.Set(startTime) - initialPause := 10 * time.Millisecond - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - mockClient := testutil.NewMockClient( - map[string]testutil.DelayedConnectReturn{ - "foo": {Err: nil, Delay: time.Millisecond * 50}, - "baz": {Err: nil, Delay: time.Millisecond * 75}, - "bar": {Err: nil, Delay: time.Millisecond * 100}, - }, - map[string]testutil.DelayedClientReturn{ - "foo": {ResultErr: errors.New("Nope"), Delay: time.Millisecond * 150}, - "bar": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBar, Size: 2}, Delay: time.Millisecond * 200}, - "baz": {ResultStats: &types.RetrievalStats{StorageProviderId: peerBaz, Size: 2}, Delay: time.Millisecond * 200}, - }, - clock, - ) - - expectedSequence := []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - ReceivedConnections: []peer.ID{peerFoo, peerBar, peerBaz}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: time.Millisecond * 50, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*50), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerFoo, Duration: time.Millisecond * 50}, - }, - }, - { - AfterStart: time.Millisecond*50 + initialPause, - ReceivedRetrievals: []peer.ID{peerFoo}, - }, - { - AfterStart: time.Millisecond * 75, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*75), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBaz}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBaz, Duration: time.Millisecond * 75}, - }, - }, - { - AfterStart: time.Millisecond * 100, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(time.Millisecond*100), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerBar}}, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerBar, Duration: time.Millisecond * 100}, - }, - }, - { - AfterStart: time.Millisecond*200 + initialPause, - ReceivedRetrievals: []peer.ID{peerBar}, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*200+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}), - events.FailedRetrieval(startTime.Add(time.Millisecond*200+initialPause), retrievalID, types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerFoo}}, multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: Nope"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerFoo}, - }, - }, - { - AfterStart: time.Millisecond*400 + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(time.Millisecond*400+initialPause), retrievalID, types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: true, FastRetrieval: false})), - events.Accepted(startTime.Add(time.Millisecond*400+initialPause), retrievalID, types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: true, FastRetrieval: false})), - events.FirstByte(startTime.Add(time.Millisecond*400+initialPause), retrievalID, types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: true, FastRetrieval: false}), time.Millisecond*200, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(time.Millisecond*400+initialPause), retrievalID, types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: true, FastRetrieval: false}), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(time.Millisecond*400+initialPause), retrievalID, types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: true, FastRetrieval: false}), 2, 0, 0, multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerBar, Duration: time.Millisecond * 200}, - {Type: testutil.SessionMetric_Success, Provider: peerBar, Value: 2}, - }, - }, - } - - // we're testing the actual Session implementation, but we also want - // to observe, so MockSession WithActual lets us do that. - scfg := session.DefaultConfig(). - WithDefaultProviderConfig(session.ProviderConfig{ - RetrievalTimeout: time.Second, - }). - WithConnectTimeAlpha(0.0). // only use the last connect time - WithoutRandomness() - _session := session.NewSession(scfg, true) - session := testutil.NewMockSession(ctx) - session.WithActual(_session) - session.RegisterRetrieval(retrievalID, cid.Undef, basicnode.NewBool(true)) - session.AddToRetrieval(retrievalID, []peer.ID{peerFoo, peerBar, peerBaz}) - - cfg := retriever.NewGraphsyncRetrieverWithConfig(session, mockClient, clock, initialPause, true) - - results := testutil.RetrievalVerifier{ - ExpectedSequence: expectedSequence, - }.RunWithVerification(ctx, t, clock, mockClient, nil, session, nil, 0, []testutil.RunRetrieval{ - func(cb func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - return cfg.Retrieve(context.Background(), types.RetrievalRequest{ - Request: trustlessutils.Request{Root: cid1}, - RetrievalID: retrievalID, - LinkSystem: cidlink.DefaultLinkSystem(), - }, cb).RetrieveFromAsyncCandidates(makeAsyncCandidates(t, []types.RetrievalCandidate{ - types.NewRetrievalCandidate(peerFoo, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: false, FastRetrieval: false}), - types.NewRetrievalCandidate(peerBaz, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: false, FastRetrieval: false}), - types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: false, FastRetrieval: false}), - types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: false, FastRetrieval: true}), - types.NewRetrievalCandidate(peerBar, nil, cid.Undef, &metadata.GraphsyncFilecoinV1{PieceCID: cid.Cid{}, VerifiedDeal: true, FastRetrieval: false}), - })) - }, - }) - require.Len(t, results, 1) - stats, err := results[0].Stats, results[0].Err - require.NoError(t, err) - require.NotNil(t, stats) - // make sure we got the final retrieval we wanted - require.Equal(t, mockClient.GetRetrievalReturns()["bar"].ResultStats, stats) -} - -func makeAsyncCandidates(t *testing.T, candidates []types.RetrievalCandidate) types.InboundAsyncCandidates { - incoming, outgoing := types.MakeAsyncCandidates(len(candidates)) - for _, candidate := range candidates { - err := outgoing.SendNext(context.Background(), []types.RetrievalCandidate{candidate}) - require.NoError(t, err) - } - close(outgoing) - return incoming -} diff --git a/pkg/retriever/httpretriever.go b/pkg/retriever/httpretriever.go index ac675657..a8b41223 100644 --- a/pkg/retriever/httpretriever.go +++ b/pkg/retriever/httpretriever.go @@ -9,7 +9,6 @@ import ( "time" "github.com/filecoin-project/go-clock" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lassie/pkg/build" "github.com/filecoin-project/lassie/pkg/events" "github.com/filecoin-project/lassie/pkg/types" @@ -97,7 +96,6 @@ func (ph *ProtocolHttp) Retrieve( ctx context.Context, retrieval *retrieval, shared *retrievalShared, - timeout time.Duration, candidate types.RetrievalCandidate, ) (*types.RetrievalStats, error) { // Connect and read body in one flow, we can move ph.beginRequest() to Connect() @@ -126,7 +124,7 @@ func (ph *ProtocolHttp) Retrieve( var ttfb time.Duration rdr := newTimeToFirstByteReader(resp.Body, func() { ttfb = retrieval.Clock.Since(retrievalStart) - shared.sendEvent(ctx, events.FirstByte(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate, ttfb, multicodec.TransportIpfsGatewayHttp)) + shared.sendEvent(ctx, candidate.MinerPeer.ID, events.FirstByte(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate, ttfb, multicodec.TransportIpfsGatewayHttp)) }) cfg := traversal.Config{ Root: retrieval.request.Root, @@ -138,7 +136,7 @@ func (ph *ProtocolHttp) Retrieve( WriteDuplicatesOut: expectDuplicates, MaxBlocks: retrieval.request.MaxBlocks, OnBlockIn: func(read uint64) { - shared.sendEvent(ctx, events.BlockReceived(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate, multicodec.TransportIpfsGatewayHttp, read)) + shared.sendEvent(ctx, candidate.MinerPeer.ID, events.BlockReceived(retrieval.Clock.Now(), retrieval.request.RetrievalID, candidate, multicodec.TransportIpfsGatewayHttp, read)) }, } @@ -157,10 +155,8 @@ func (ph *ProtocolHttp) Retrieve( Blocks: traversalResult.BlocksIn, Duration: duration, AverageSpeed: speed, - TotalPayment: big.Zero(), - NumPayments: 0, - AskPrice: big.Zero(), TimeToFirstByte: ttfb, + Providers: []string{candidate.Endpoint()}, }, nil } @@ -177,7 +173,7 @@ func (ph *ProtocolHttp) beginRequest(ctx context.Context, request types.Retrieva func makeRequest(ctx context.Context, request types.RetrievalRequest, candidate types.RetrievalCandidate) (*http.Request, error) { candidateURL, err := candidate.ToURL() if err != nil { - logger.Warnf("Couldn't construct a url for miner %s: %v", candidate.MinerPeer.ID, err) + logger.Warnf("Couldn't construct a url for provider %s: %v", candidate.Endpoint(), err) return nil, fmt.Errorf("%w: %v", ErrNoHttpForPeer, err) } @@ -190,8 +186,8 @@ func makeRequest(ctx context.Context, request types.RetrievalRequest, candidate reqURL := fmt.Sprintf("%s/ipfs/%s%s", candidateURL, request.Root, path) req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { - logger.Warnf("Couldn't construct a http request %s: %v", candidate.MinerPeer.ID, err) - return nil, fmt.Errorf("%w for peer %s: %v", ErrBadPathForRequest, candidate.MinerPeer.ID, err) + logger.Warnf("Couldn't construct http request for %s: %v", candidateURL, err) + return nil, fmt.Errorf("%w for %s: %v", ErrBadPathForRequest, candidateURL, err) } req.Header.Add("Accept", trustlesshttp.DefaultContentType().String()) // prefer duplicates req.Header.Add("X-Request-Id", request.RetrievalID.String()) diff --git a/pkg/retriever/hybridretriever.go b/pkg/retriever/hybridretriever.go new file mode 100644 index 00000000..3a71acbf --- /dev/null +++ b/pkg/retriever/hybridretriever.go @@ -0,0 +1,578 @@ +package retriever + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/filecoin-project/go-clock" + "github.com/filecoin-project/lassie/pkg/blockbroker" + "github.com/filecoin-project/lassie/pkg/events" + "github.com/filecoin-project/lassie/pkg/extractor" + "github.com/filecoin-project/lassie/pkg/types" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-trustless-utils/traversal" +) + +const DefaultRawLeafConcurrency = 8 + +// HybridRetriever tries a full CAR fetch first, then falls back to +// per-block frontier traversal when the initial response is incomplete. +type HybridRetriever struct { + inner types.Retriever + candidateSource types.CandidateSource + httpClient *http.Client + clock clock.Clock + skipBlockVerification bool +} + +func NewHybridRetriever( + inner types.Retriever, + candidateSource types.CandidateSource, + httpClient *http.Client, + skipBlockVerification bool, +) *HybridRetriever { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &HybridRetriever{ + inner: inner, + candidateSource: candidateSource, + httpClient: httpClient, + clock: clock.New(), + skipBlockVerification: skipBlockVerification, + } +} + +type phaseOneStats struct { + bytesReceived uint64 + blocksReceived uint64 +} + +func (hr *HybridRetriever) Retrieve( + ctx context.Context, + request types.RetrievalRequest, + eventsCallback func(types.RetrievalEvent), +) (*types.RetrievalStats, error) { + startTime := hr.clock.Now() + + if eventsCallback == nil { + eventsCallback = func(types.RetrievalEvent) {} + } + + var p1Stats phaseOneStats + wrappedCallback := func(event types.RetrievalEvent) { + if br, ok := event.(interface{ ByteCount() uint64 }); ok { + p1Stats.bytesReceived += br.ByteCount() + p1Stats.blocksReceived++ + } + eventsCallback(event) + } + + stats, err := hr.inner.Retrieve(ctx, request, wrappedCallback) + if err == nil { + return stats, nil + } + + missingCid, ok := extractMissingCid(err) + if !ok { + return nil, err + } + + logger.Infow("switching to per-block retrieval (missing blocks in initial response)", + "root", request.Root, + "missingCid", missingCid, + "phase1Blocks", p1Stats.blocksReceived, + "phase1Bytes", p1Stats.bytesReceived) + + stats, err = hr.continuePerBlock(ctx, request, eventsCallback, startTime, p1Stats) + if err != nil { + return nil, err + } + return stats, nil +} + +func (hr *HybridRetriever) continuePerBlock( + ctx context.Context, + request types.RetrievalRequest, + eventsCallback func(types.RetrievalEvent), + startTime time.Time, + p1Stats phaseOneStats, +) (*types.RetrievalStats, error) { + session := blockbroker.NewSession(hr.candidateSource, hr.httpClient, hr.skipBlockVerification) + defer session.Close() + + session.SeedProviders(ctx, request.Root) + + var p2BlocksOut uint64 + var p2BytesOut uint64 + + if request.MaxBlocks > 0 && p1Stats.blocksReceived >= request.MaxBlocks { + logger.Infow("max blocks limit already reached in phase 1", "limit", request.MaxBlocks) + duration := hr.clock.Since(startTime) + var speed uint64 + if duration.Seconds() > 0 { + speed = uint64(float64(p1Stats.bytesReceived) / duration.Seconds()) + } + return &types.RetrievalStats{ + RootCid: request.Root, + Size: p1Stats.bytesReceived, + Blocks: p1Stats.blocksReceived, + Duration: duration, + AverageSpeed: speed, + Providers: session.UsedProviders(), + }, nil + } + + var maxBlocks uint64 + if request.MaxBlocks > 0 { + maxBlocks = request.MaxBlocks - p1Stats.blocksReceived + } + + err := hr.streamingTraverse(ctx, request, session, &p2BytesOut, &p2BlocksOut, maxBlocks) + if err != nil { + return nil, fmt.Errorf("per-block traversal failed: %w", err) + } + + totalBytes := p1Stats.bytesReceived + atomic.LoadUint64(&p2BytesOut) + totalBlocks := p1Stats.blocksReceived + atomic.LoadUint64(&p2BlocksOut) + duration := hr.clock.Since(startTime) + + var speed uint64 + if duration.Seconds() > 0 { + speed = uint64(float64(totalBytes) / duration.Seconds()) + } + + return &types.RetrievalStats{ + RootCid: request.Root, + Size: totalBytes, + Blocks: totalBlocks, + Duration: duration, + AverageSpeed: speed, + Providers: session.UsedProviders(), + }, nil +} + +// streamingTraverse fetches blocks via frontier DFS, writing each to output +// immediately. Raw leaves are batched and fetched in parallel. +func (hr *HybridRetriever) streamingTraverse( + ctx context.Context, + request types.RetrievalRequest, + session blockbroker.BlockSession, + bytesOut *uint64, + blocksOut *uint64, + maxBlocks uint64, +) error { + frontier := NewFrontier(request.Root) + baseLsys := request.LinkSystem + + var blockCount uint64 + + for !frontier.Empty() { + if ctx.Err() != nil { + return ctx.Err() + } + + c := frontier.Pop() + + if frontier.Seen(c) { + continue + } + + // check phase 1 cache + if baseLsys.StorageReadOpener != nil { + rdr, err := baseLsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err == nil { + data, readErr := io.ReadAll(rdr) + if readErr != nil { + logger.Warnw("cached block read failed, fetching from network", "cid", c, "err", readErr) + } else { + block, blockErr := blocks.NewBlockWithCid(data, c) + if blockErr != nil { + logger.Warnw("cached block parse failed, fetching from network", "cid", c, "err", blockErr) + } else { + logger.Debugw("using cached block from phase 1", "cid", c) + links, _ := ExtractLinks(block) + frontier.PushAll(links) + frontier.MarkSeen(c) + continue + } + } + } + } + + block, err := hr.fetchBlock(ctx, c, session, baseLsys) + if err != nil { + return fmt.Errorf("failed to fetch block %s: %w", c, err) + } + + links, err := ExtractLinks(block) + if err != nil { + logger.Warnw("failed to extract links", "cid", c, "err", err) + } else { + rawLeaves, dagNodes := separateLinksByCodec(ctx, links, frontier, baseLsys) + frontier.PushAll(dagNodes) + + if maxBlocks > 0 && uint64(len(rawLeaves)) > maxBlocks-blockCount { + rawLeaves = rawLeaves[:maxBlocks-blockCount] + } + if len(rawLeaves) > 0 { + fetchedBlocks, fetchErr := hr.parallelFetchRawLeaves(ctx, rawLeaves, session) + if fetchErr != nil { + return fmt.Errorf("parallel fetch failed: %w", fetchErr) + } + + for _, b := range fetchedBlocks { + if err := hr.writeBlock(ctx, b, baseLsys); err != nil { + return fmt.Errorf("failed to write block %s: %w", b.Cid(), err) + } + atomic.AddUint64(bytesOut, uint64(len(b.RawData()))) + atomic.AddUint64(blocksOut, 1) + frontier.MarkSeen(b.Cid()) + blockCount++ + + if maxBlocks > 0 && blockCount >= maxBlocks { + logger.Infow("reached max blocks limit", "limit", maxBlocks) + return nil + } + } + } + } + + if err := hr.writeBlock(ctx, block, baseLsys); err != nil { + return fmt.Errorf("failed to write block %s: %w", c, err) + } + + blockData := block.RawData() + atomic.AddUint64(bytesOut, uint64(len(blockData))) + atomic.AddUint64(blocksOut, 1) + + frontier.MarkSeen(c) + blockCount++ + + if maxBlocks > 0 && blockCount >= maxBlocks { + logger.Infow("reached max blocks limit", "limit", maxBlocks) + break + } + } + + return nil +} + +// separateLinksByCodec splits CIDs into raw leaves (codec 0x55, no children, +// safe to fetch in parallel) and dag nodes. Skips already-seen CIDs and +// raw leaves already in storage. +func separateLinksByCodec(ctx context.Context, links []cid.Cid, frontier *Frontier, lsys linking.LinkSystem) (rawLeaves, dagNodes []cid.Cid) { + for _, c := range links { + if frontier.Seen(c) { + continue + } + if c.Prefix().Codec == cid.Raw { + if lsys.StorageReadOpener != nil { + _, err := lsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err == nil { + frontier.MarkSeen(c) + logger.Debugw("raw leaf already in storage, skipping fetch", "cid", c) + continue + } + } + rawLeaves = append(rawLeaves, c) + } else { + dagNodes = append(dagNodes, c) + } + } + return +} + +func (hr *HybridRetriever) parallelFetchRawLeaves( + ctx context.Context, + cids []cid.Cid, + session blockbroker.BlockSession, +) ([]blocks.Block, error) { + if len(cids) == 0 { + return nil, nil + } + + logger.Debugw("parallel fetching raw leaves", "count", len(cids)) + + results := make([]blocks.Block, len(cids)) + var firstErr error + var errOnce sync.Once + var wg sync.WaitGroup + + sem := make(chan struct{}, DefaultRawLeafConcurrency) + + for i, c := range cids { + wg.Add(1) + go func(idx int, c cid.Cid) { + defer wg.Done() + + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + errOnce.Do(func() { firstErr = ctx.Err() }) + return + } + + block, err := session.Get(ctx, c) + if err != nil { + errOnce.Do(func() { firstErr = fmt.Errorf("fetch %s: %w", c, err) }) + return + } + results[idx] = block + }(i, c) + } + + wg.Wait() + + if firstErr != nil { + return nil, firstErr + } + + logger.Debugw("parallel fetch complete", "fetched", len(results)) + return results, nil +} + +// fetchBlock prefers CAR subgraph for dag-pb/dag-cbor (may fetch children too), +// falls back to raw single-block fetch. Raw codec leaves skip CAR entirely. +func (hr *HybridRetriever) fetchBlock( + ctx context.Context, + c cid.Cid, + session blockbroker.BlockSession, + baseLsys linking.LinkSystem, +) (blocks.Block, error) { + if c.Prefix().Codec == cid.Raw { + block, err := session.Get(ctx, c) + if err != nil { + return nil, err + } + logger.Debugw("fetched raw leaf block", "cid", c, "bytes", len(block.RawData())) + return block, nil + } + + if baseLsys.StorageReadOpener != nil { + blocksFromCAR, carErr := session.GetSubgraph(ctx, c, baseLsys) + if carErr == nil && blocksFromCAR > 0 { + logger.Debugw("fetched subgraph via CAR", "cid", c, "blocks", blocksFromCAR) + rdr, err := baseLsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err == nil { + data, err := io.ReadAll(rdr) + if err == nil { + return blocks.NewBlockWithCid(data, c) + } + logger.Warnw("CAR readback failed after successful fetch", "cid", c, "err", err) + } else { + logger.Warnw("CAR storage read failed after successful fetch", "cid", c, "err", err) + } + } else if carErr != nil { + logger.Debugw("CAR subgraph unavailable, trying single block", "cid", c, "reason", carErr) + } + } + + block, err := session.Get(ctx, c) + if err != nil { + return nil, err + } + logger.Debugw("fetched single block", "cid", c, "bytes", len(block.RawData())) + return block, nil +} + +func (hr *HybridRetriever) writeBlock( + ctx context.Context, + block blocks.Block, + baseLsys linking.LinkSystem, +) error { + if baseLsys.StorageWriteOpener == nil { + return nil + } + + w, wc, err := baseLsys.StorageWriteOpener(linking.LinkContext{Ctx: ctx}) + if err != nil { + return err + } + + if _, err := io.Copy(w, bytes.NewReader(block.RawData())); err != nil { + return err + } + + return wc(cidlink.Link{Cid: block.Cid()}) +} + +func extractMissingCid(err error) (cid.Cid, bool) { + var notFound format.ErrNotFound + if errors.As(err, ¬Found) { + return notFound.Cid, true + } + + if errors.Is(err, traversal.ErrMissingBlock) { + var nf format.ErrNotFound + if errors.As(err, &nf) { + return nf.Cid, true + } + return cid.Undef, true + } + + return cid.Undef, false +} + +// RetrieveAndExtract retrieves content and extracts UnixFS to disk in a +// single pass without buffering entire files in memory. +func (hr *HybridRetriever) RetrieveAndExtract( + ctx context.Context, + rootCid cid.Cid, + ext *extractor.Extractor, + eventsCallback func(types.RetrievalEvent), + onBlock func(int), +) (*types.RetrievalStats, error) { + startTime := hr.clock.Now() + if eventsCallback == nil { + eventsCallback = func(types.RetrievalEvent) {} + } + + session := blockbroker.NewSession(hr.candidateSource, hr.httpClient, hr.skipBlockVerification) + defer session.Close() + + session.SeedProviders(ctx, rootCid) + + carReader := extractor.NewExtractingCarReader(ext, rootCid) + if onBlock != nil { + carReader.OnBlock(onBlock) + } + + var totalBlocks uint64 + var totalBytes uint64 + var primaryProvider string + + rdr, provider, err := session.GetSubgraphStream(ctx, rootCid) + if err == nil { + primaryProvider = provider + eventsCallback(events.ExtractionStarted(hr.clock.Now(), rootCid, provider)) + logger.Infow("streaming CAR extraction", "root", rootCid, "provider", provider) + + blocks, bytes, extractErr := carReader.ReadAndExtract(ctx, rdr) + rdr.Close() + + totalBlocks = blocks + totalBytes = bytes + + if extractErr == nil { + duration := hr.clock.Since(startTime) + var speed uint64 + if duration.Seconds() > 0 { + speed = uint64(float64(totalBytes) / duration.Seconds()) + } + eventsCallback(events.ExtractionSucceeded(hr.clock.Now(), rootCid, provider, totalBytes, totalBlocks, duration)) + return &types.RetrievalStats{ + RootCid: rootCid, + Size: totalBytes, + Blocks: totalBlocks, + Duration: duration, + AverageSpeed: speed, + Providers: session.UsedProviders(), + }, nil + } + + missing, isMissing := extractor.IsMissing(extractErr) + if !isMissing { + return nil, fmt.Errorf("extraction failed: %w", extractErr) + } + + logger.Infow("CAR incomplete, falling back to per-block", + "root", rootCid, + "blocksFromCAR", totalBlocks, + "missing", len(missing)) + + p2Blocks, p2Bytes, err := hr.extractPerBlock(ctx, ext, session, missing, onBlock) + if err != nil { + return nil, fmt.Errorf("per-block extraction failed: %w", err) + } + totalBlocks += p2Blocks + totalBytes += p2Bytes + } else { + logger.Infow("CAR unavailable, using per-block extraction", "root", rootCid, "err", err) + eventsCallback(events.ExtractionStarted(hr.clock.Now(), rootCid, "per-block")) + primaryProvider = "per-block" + + p2Blocks, p2Bytes, err := hr.extractPerBlock(ctx, ext, session, []cid.Cid{rootCid}, onBlock) + if err != nil { + return nil, fmt.Errorf("per-block extraction failed: %w", err) + } + totalBlocks = p2Blocks + totalBytes = p2Bytes + } + + duration := hr.clock.Since(startTime) + var speed uint64 + if duration.Seconds() > 0 { + speed = uint64(float64(totalBytes) / duration.Seconds()) + } + + eventsCallback(events.ExtractionSucceeded(hr.clock.Now(), rootCid, primaryProvider, totalBytes, totalBlocks, duration)) + return &types.RetrievalStats{ + RootCid: rootCid, + Size: totalBytes, + Blocks: totalBlocks, + Duration: duration, + AverageSpeed: speed, + Providers: session.UsedProviders(), + }, nil +} + +func (hr *HybridRetriever) extractPerBlock( + ctx context.Context, + ext *extractor.Extractor, + session blockbroker.BlockSession, + startCids []cid.Cid, + onBlock func(int), +) (uint64, uint64, error) { + frontier := NewFrontier(cid.Undef) + frontier.pending = append(frontier.pending, startCids...) + + var totalBlocks uint64 + var totalBytes uint64 + + for !frontier.Empty() { + if ctx.Err() != nil { + return totalBlocks, totalBytes, ctx.Err() + } + + c := frontier.Pop() + if frontier.Seen(c) { + continue + } + + block, err := session.Get(ctx, c) + if err != nil { + return totalBlocks, totalBytes, fmt.Errorf("failed to fetch block %s: %w", c, err) + } + + children, err := ext.ProcessBlock(ctx, block) + if err != nil { + logger.Warnw("extraction failed for block", "cid", c, "err", err) + } + + frontier.PushAll(children) + frontier.MarkSeen(c) + + blockSize := len(block.RawData()) + totalBlocks++ + totalBytes += uint64(blockSize) + + if onBlock != nil { + onBlock(blockSize) + } + } + + return totalBlocks, totalBytes, nil +} diff --git a/pkg/retriever/hybridretriever_test.go b/pkg/retriever/hybridretriever_test.go new file mode 100644 index 00000000..a1d2792b --- /dev/null +++ b/pkg/retriever/hybridretriever_test.go @@ -0,0 +1,652 @@ +package retriever + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/fluent/qp" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/storage/memstore" + trustlessutils "github.com/ipld/go-trustless-utils" + "github.com/ipni/go-libipni/metadata" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +func TestCrossProviderDAGConstruction(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Build a simple DAG: root -> [child1, child2, child3] + // Provider A will have: root, child1 + // Provider B will have: child2, child3 + store := &memstore.Store{} + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(store) + lsys.SetReadStorage(store) + + lp := cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagCbor), + MhType: multihash.SHA2_256, + MhLength: 32, + }, + } + + child1, child1Bytes := makeLeafNode(t, lsys, lp, "leaf-data-1") + child2, child2Bytes := makeLeafNode(t, lsys, lp, "leaf-data-2") + child3, child3Bytes := makeLeafNode(t, lsys, lp, "leaf-data-3") + + rootNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { + qp.MapEntry(ma, "children", qp.List(-1, func(la datamodel.ListAssembler) { + qp.ListEntry(la, qp.Link(child1)) + qp.ListEntry(la, qp.Link(child2)) + qp.ListEntry(la, qp.Link(child3)) + })) + }) + require.NoError(t, err) + + rootLink, err := lsys.Store(linking.LinkContext{Ctx: ctx}, lp, rootNode) + require.NoError(t, err) + rootCid := rootLink.(cidlink.Link).Cid + + rootBytes := getBlockBytes(t, store, rootCid) + + // provider A: root + child1; provider B: child2 + child3 + providerABlocks := map[cid.Cid][]byte{ + rootCid: rootBytes, + child1.(cidlink.Link).Cid: child1Bytes, + } + + providerBBlocks := map[cid.Cid][]byte{ + child2.(cidlink.Link).Cid: child2Bytes, + child3.(cidlink.Link).Cid: child3Bytes, + } + + serverA := httptest.NewServer(makeBlockHandler(t, providerABlocks, "providerA")) + defer serverA.Close() + + serverB := httptest.NewServer(makeBlockHandler(t, providerBBlocks, "providerB")) + defer serverB.Close() + + candidateA := makeMockCandidate(t, "provider-a", serverA.URL, rootCid, "12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4") + candidateB := makeMockCandidate(t, "provider-b", serverB.URL, rootCid, "12D3KooWRYqH1rDGfPMQaqaELfEBJXBnkSvRMxNR4AjqE7yXKqLH") + + mockSource := &mockCandidateSource{ + candidates: map[cid.Cid][]types.RetrievalCandidate{ + rootCid: {candidateA, candidateB}, + child1.(cidlink.Link).Cid: {candidateA}, + child2.(cidlink.Link).Cid: {candidateB}, + child3.(cidlink.Link).Cid: {candidateB}, + }, + } + + // inner retriever fails on child2 — simulates partial whole-DAG fetch + mockInner := &mockFailingRetriever{ + failWithCid: child2.(cidlink.Link).Cid, + } + + outputStore := &memstore.Store{} + outputLsys := cidlink.DefaultLinkSystem() + outputLsys.SetWriteStorage(outputStore) + outputLsys.SetReadStorage(outputStore) + + // pre-populate with blocks fetched before the failure point + outputStore.Put(ctx, rootCid.KeyString(), rootBytes) + outputStore.Put(ctx, child1.(cidlink.Link).Cid.KeyString(), child1Bytes) + + hr := NewHybridRetriever(mockInner, mockSource, http.DefaultClient, false) + + request := types.RetrievalRequest{ + Request: trustlessutils.Request{ + Root: rootCid, + Scope: trustlessutils.DagScopeAll, + }, + LinkSystem: outputLsys, + } + + stats, err := hr.Retrieve(ctx, request, nil) + require.NoError(t, err) + require.NotNil(t, stats) + + verifyBlockExists(t, outputStore, ctx, rootCid) + verifyBlockExists(t, outputStore, ctx, child1.(cidlink.Link).Cid) + verifyBlockExists(t, outputStore, ctx, child2.(cidlink.Link).Cid) + verifyBlockExists(t, outputStore, ctx, child3.(cidlink.Link).Cid) + + t.Logf("Successfully retrieved DAG across providers: %d blocks, %d bytes", + stats.Blocks, stats.Size) +} + +func TestCrossProviderDeepDAG(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Build a deeper DAG: root -> mid -> [leaf1, leaf2] + store := &memstore.Store{} + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(store) + lsys.SetReadStorage(store) + + lp := cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagCbor), + MhType: multihash.SHA2_256, + MhLength: 32, + }, + } + + leaf1, leaf1Bytes := makeLeafNode(t, lsys, lp, "deep-leaf-1") + leaf2, leaf2Bytes := makeLeafNode(t, lsys, lp, "deep-leaf-2") + + midNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { + qp.MapEntry(ma, "leaves", qp.List(-1, func(la datamodel.ListAssembler) { + qp.ListEntry(la, qp.Link(leaf1)) + qp.ListEntry(la, qp.Link(leaf2)) + })) + }) + require.NoError(t, err) + + midLink, err := lsys.Store(linking.LinkContext{Ctx: ctx}, lp, midNode) + require.NoError(t, err) + midCid := midLink.(cidlink.Link).Cid + midBytes := getBlockBytes(t, store, midCid) + + rootNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { + qp.MapEntry(ma, "child", qp.Link(midLink)) + }) + require.NoError(t, err) + + rootLink, err := lsys.Store(linking.LinkContext{Ctx: ctx}, lp, rootNode) + require.NoError(t, err) + rootCid := rootLink.(cidlink.Link).Cid + rootBytes := getBlockBytes(t, store, rootCid) + + // Provider A: has root and mid only (directory structure) + providerABlocks := map[cid.Cid][]byte{ + rootCid: rootBytes, + midCid: midBytes, + } + + // Provider B: has the leaves + providerBBlocks := map[cid.Cid][]byte{ + leaf1.(cidlink.Link).Cid: leaf1Bytes, + leaf2.(cidlink.Link).Cid: leaf2Bytes, + } + + serverA := httptest.NewServer(makeBlockHandler(t, providerABlocks, "providerA")) + defer serverA.Close() + + serverB := httptest.NewServer(makeBlockHandler(t, providerBBlocks, "providerB")) + defer serverB.Close() + + candidateA := makeMockCandidate(t, "provider-a", serverA.URL, rootCid, "12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4") + candidateB := makeMockCandidate(t, "provider-b", serverB.URL, rootCid, "12D3KooWRYqH1rDGfPMQaqaELfEBJXBnkSvRMxNR4AjqE7yXKqLH") + + mockSource := &mockCandidateSource{ + candidates: map[cid.Cid][]types.RetrievalCandidate{ + rootCid: {candidateA}, + midCid: {candidateA}, + leaf1.(cidlink.Link).Cid: {candidateB}, + leaf2.(cidlink.Link).Cid: {candidateB}, + }, + } + + mockInner := &mockFailingRetriever{ + failWithCid: leaf1.(cidlink.Link).Cid, + } + + outputStore := &memstore.Store{} + outputLsys := cidlink.DefaultLinkSystem() + outputLsys.SetWriteStorage(outputStore) + outputLsys.SetReadStorage(outputStore) + + // Pre-populate with partial fetch + outputStore.Put(ctx, rootCid.KeyString(), rootBytes) + outputStore.Put(ctx, midCid.KeyString(), midBytes) + + hr := NewHybridRetriever(mockInner, mockSource, http.DefaultClient, false) + + request := types.RetrievalRequest{ + Request: trustlessutils.Request{ + Root: rootCid, + Scope: trustlessutils.DagScopeAll, + }, + LinkSystem: outputLsys, + } + + stats, err := hr.Retrieve(ctx, request, nil) + require.NoError(t, err) + require.NotNil(t, stats) + + verifyBlockExists(t, outputStore, ctx, rootCid) + verifyBlockExists(t, outputStore, ctx, midCid) + verifyBlockExists(t, outputStore, ctx, leaf1.(cidlink.Link).Cid) + verifyBlockExists(t, outputStore, ctx, leaf2.(cidlink.Link).Cid) + + t.Logf("Successfully retrieved deep DAG: %d blocks", stats.Blocks) +} + +func TestNoFallbackOnNonMissingError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCid := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") + + mockInner := &mockFailingRetriever{ + failWithErr: errors.New("network timeout"), + } + + mockSource := &mockCandidateSource{ + candidates: map[cid.Cid][]types.RetrievalCandidate{}, + } + + hr := NewHybridRetriever(mockInner, mockSource, http.DefaultClient, false) + + outputStore := &memstore.Store{} + outputLsys := cidlink.DefaultLinkSystem() + outputLsys.SetWriteStorage(outputStore) + outputLsys.SetReadStorage(outputStore) + + request := types.RetrievalRequest{ + Request: trustlessutils.Request{ + Root: rootCid, + Scope: trustlessutils.DagScopeAll, + }, + LinkSystem: outputLsys, + } + + _, err := hr.Retrieve(ctx, request, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "network timeout") +} + +// P1 has root→node1, P2 has node1→node2→node3→node4→leaf. +// when P1 fails on node1, P2's subgraph should come as one CAR request. +func TestEfficientSubgraphRetrieval(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + store := &memstore.Store{} + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(store) + lsys.SetReadStorage(store) + + lp := cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagCbor), + MhType: multihash.SHA2_256, + MhLength: 32, + }, + } + + // Build a CHAIN: root → node1 → node2 → node3 → node4 → leaf + // This way when we fetch node1 as CAR, we get the entire subgraph + leaf, leafBytes := makeLeafNode(t, lsys, lp, "leaf-data") + + // Build chain bottom-up + node4, node4Bytes := makeChainNode(t, lsys, lp, "node4", leaf) + node3, node3Bytes := makeChainNode(t, lsys, lp, "node3", node4) + node2, node2Bytes := makeChainNode(t, lsys, lp, "node2", node3) + node1, node1Bytes := makeChainNode(t, lsys, lp, "node1", node2) + + rootNode, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { + qp.MapEntry(ma, "child", qp.Link(node1)) + }) + require.NoError(t, err) + + rootLink, err := lsys.Store(linking.LinkContext{Ctx: ctx}, lp, rootNode) + require.NoError(t, err) + rootCid := rootLink.(cidlink.Link).Cid + rootBytes := getBlockBytes(t, store, rootCid) + + // Provider A: has only root (will fail on node1) + providerABlocks := map[cid.Cid][]byte{ + rootCid: rootBytes, + } + + // Provider B: has the entire chain starting from node1 + providerBBlocks := map[cid.Cid][]byte{ + node1.(cidlink.Link).Cid: node1Bytes, + node2.(cidlink.Link).Cid: node2Bytes, + node3.(cidlink.Link).Cid: node3Bytes, + node4.(cidlink.Link).Cid: node4Bytes, + leaf.(cidlink.Link).Cid: leafBytes, + } + + var providerARequests int32 + var providerBRawRequests int32 + var providerBCARRequests int32 + + serverA := httptest.NewServer(makeBlockHandlerWithStats(t, providerABlocks, "providerA", &providerARequests)) + defer serverA.Close() + + serverB := httptest.NewServer(makeHybridHandler(t, providerBBlocks, "providerB", &providerBRawRequests, &providerBCARRequests)) + defer serverB.Close() + + candidateA := makeMockCandidate(t, "provider-a", serverA.URL, rootCid, "12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4") + candidateB := makeMockCandidate(t, "provider-b", serverB.URL, rootCid, "12D3KooWRYqH1rDGfPMQaqaELfEBJXBnkSvRMxNR4AjqE7yXKqLH") + + mockSource := &mockCandidateSource{ + candidates: map[cid.Cid][]types.RetrievalCandidate{ + rootCid: {candidateA, candidateB}, + node1.(cidlink.Link).Cid: {candidateB}, + node2.(cidlink.Link).Cid: {candidateB}, + node3.(cidlink.Link).Cid: {candidateB}, + node4.(cidlink.Link).Cid: {candidateB}, + leaf.(cidlink.Link).Cid: {candidateB}, + }, + } + + mockInner := &mockFailingRetriever{ + failWithCid: node1.(cidlink.Link).Cid, + } + + outputStore := &memstore.Store{} + outputLsys := cidlink.DefaultLinkSystem() + outputLsys.SetWriteStorage(outputStore) + outputLsys.SetReadStorage(outputStore) + + // Pre-populate with root from partial fetch + outputStore.Put(ctx, rootCid.KeyString(), rootBytes) + + hr := NewHybridRetriever(mockInner, mockSource, http.DefaultClient, false) + + request := types.RetrievalRequest{ + Request: trustlessutils.Request{ + Root: rootCid, + Scope: trustlessutils.DagScopeAll, + }, + LinkSystem: outputLsys, + } + + stats, err := hr.Retrieve(ctx, request, nil) + require.NoError(t, err) + require.NotNil(t, stats) + + verifyBlockExists(t, outputStore, ctx, rootCid) + verifyBlockExists(t, outputStore, ctx, node1.(cidlink.Link).Cid) + verifyBlockExists(t, outputStore, ctx, node2.(cidlink.Link).Cid) + verifyBlockExists(t, outputStore, ctx, node3.(cidlink.Link).Cid) + verifyBlockExists(t, outputStore, ctx, node4.(cidlink.Link).Cid) + verifyBlockExists(t, outputStore, ctx, leaf.(cidlink.Link).Cid) + + carRequests := atomic.LoadInt32(&providerBCARRequests) + rawRequests := atomic.LoadInt32(&providerBRawRequests) + + t.Logf("Provider B: CAR requests=%d, raw requests=%d (chain of 5 blocks)", carRequests, rawRequests) + + // Key assertion: With a chain, ONE CAR request for node1 should fetch all 5 blocks + // Without efficient CAR fetching, we'd need 5 separate requests + require.GreaterOrEqual(t, carRequests, int32(1), "should have made at least one CAR request") + + if carRequests == 1 && rawRequests == 0 { + t.Logf("OPTIMAL: Fetched entire chain (5 blocks) with 1 CAR request!") + } else { + t.Logf("Note: CAR=%d, raw=%d (could be optimized)", carRequests, rawRequests) + } +} + +func makeLeafNode(t *testing.T, lsys linking.LinkSystem, lp cidlink.LinkPrototype, data string) (datamodel.Link, []byte) { + node, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { + qp.MapEntry(ma, "data", qp.String(data)) + }) + require.NoError(t, err) + + link, err := lsys.Store(linking.LinkContext{}, lp, node) + require.NoError(t, err) + + var buf bytes.Buffer + err = dagcbor.Encode(node, &buf) + require.NoError(t, err) + + return link, buf.Bytes() +} + +func makeChainNode(t *testing.T, lsys linking.LinkSystem, lp cidlink.LinkPrototype, name string, childLink datamodel.Link) (datamodel.Link, []byte) { + node, err := qp.BuildMap(basicnode.Prototype.Any, -1, func(ma datamodel.MapAssembler) { + qp.MapEntry(ma, "name", qp.String(name)) + qp.MapEntry(ma, "child", qp.Link(childLink)) + }) + require.NoError(t, err) + + link, err := lsys.Store(linking.LinkContext{}, lp, node) + require.NoError(t, err) + + var buf bytes.Buffer + err = dagcbor.Encode(node, &buf) + require.NoError(t, err) + + return link, buf.Bytes() +} + +func getBlockBytes(t *testing.T, store *memstore.Store, c cid.Cid) []byte { + data, err := store.Get(context.Background(), c.KeyString()) + require.NoError(t, err) + return data +} + +func makeBlockHandler(t *testing.T, blocks map[cid.Cid][]byte, name string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Parse CID from path: /ipfs/{cid} + path := r.URL.Path + if len(path) < 7 { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + cidStr := path[6:] // strip "/ipfs/" + // Handle query params + if idx := len(cidStr) - 1; idx > 0 { + for i, c := range cidStr { + if c == '?' { + cidStr = cidStr[:i] + break + } + } + } + c, err := cid.Parse(cidStr) + if err != nil { + http.Error(w, "invalid cid", http.StatusBadRequest) + return + } + + data, ok := blocks[c] + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/vnd.ipld.raw") + w.WriteHeader(http.StatusOK) + w.Write(data) + }) +} + +func makeBlockHandlerWithStats(t *testing.T, blocks map[cid.Cid][]byte, name string, requestCount *int32) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(requestCount, 1) + + path := r.URL.Path + if len(path) < 7 { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + cidStr := path[6:] + if idx := strings.Index(cidStr, "?"); idx > 0 { + cidStr = cidStr[:idx] + } + c, err := cid.Parse(cidStr) + if err != nil { + http.Error(w, "invalid cid", http.StatusBadRequest) + return + } + + data, ok := blocks[c] + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/vnd.ipld.raw") + w.WriteHeader(http.StatusOK) + w.Write(data) + }) +} + +func makeHybridHandler(t *testing.T, blocks map[cid.Cid][]byte, name string, rawCount, carCount *int32) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + if len(path) < 7 { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + cidStr := path[6:] + if idx := strings.Index(cidStr, "?"); idx > 0 { + cidStr = cidStr[:idx] + } + c, err := cid.Parse(cidStr) + if err != nil { + http.Error(w, "invalid cid", http.StatusBadRequest) + return + } + + // Check if this is a CAR request (dag-scope parameter present) + query := r.URL.Query() + if query.Get("dag-scope") != "" || strings.Contains(r.Header.Get("Accept"), "application/vnd.ipld.car") { + atomic.AddInt32(carCount, 1) + + // return all blocks we have (simulating a complete provider) + var buf bytes.Buffer + carWriter, err := storage.NewWritable(&buf, []cid.Cid{c}, car.WriteAsCarV1(true)) + if err != nil { + http.Error(w, "failed to create car writer", http.StatusInternalServerError) + return + } + + data, ok := blocks[c] + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + if err := carWriter.Put(r.Context(), c.KeyString(), data); err != nil { + http.Error(w, "failed to write block", http.StatusInternalServerError) + return + } + + for blockCid, blockData := range blocks { + if blockCid.Equals(c) { + continue // Already written + } + _ = carWriter.Put(r.Context(), blockCid.KeyString(), blockData) + } + + if err := carWriter.Finalize(); err != nil { + http.Error(w, "failed to finalize car", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/vnd.ipld.car; version=1; order=dfs; dups=y") + w.WriteHeader(http.StatusOK) + w.Write(buf.Bytes()) + return + } + + // Raw block request + atomic.AddInt32(rawCount, 1) + + data, ok := blocks[c] + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/vnd.ipld.raw") + w.WriteHeader(http.StatusOK) + w.Write(data) + }) +} + +func makeMockCandidate(t *testing.T, name string, serverURL string, rootCid cid.Cid, peerIDStr string) types.RetrievalCandidate { + // convert http://host:port to /ip4/host/tcp/port/http multiaddr + u, err := url.Parse(serverURL) + require.NoError(t, err) + host, port, err := net.SplitHostPort(u.Host) + require.NoError(t, err) + + maddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s/http", host, port)) + require.NoError(t, err) + + pid, err := peer.Decode(peerIDStr) + require.NoError(t, err) + + return types.NewRetrievalCandidate( + pid, + []multiaddr.Multiaddr{maddr}, + rootCid, + &metadata.IpfsGatewayHttp{}, + ) +} + +func verifyBlockExists(t *testing.T, store *memstore.Store, ctx context.Context, c cid.Cid) { + has, err := store.Has(ctx, c.KeyString()) + require.NoError(t, err) + require.True(t, has, "block %s should exist in store", c) +} + +type mockCandidateSource struct { + candidates map[cid.Cid][]types.RetrievalCandidate + mu sync.Mutex +} + +func (m *mockCandidateSource) FindCandidates(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error { + m.mu.Lock() + candidates := m.candidates[c] + m.mu.Unlock() + + for _, cand := range candidates { + cb(cand) + } + return nil +} + +type mockFailingRetriever struct { + failWithCid cid.Cid + failWithErr error +} + +func (m *mockFailingRetriever) Retrieve( + ctx context.Context, + request types.RetrievalRequest, + eventsCallback func(types.RetrievalEvent), +) (*types.RetrievalStats, error) { + if m.failWithErr != nil { + return nil, m.failWithErr + } + // Simulate ErrNotFound for the specific CID + return nil, format.ErrNotFound{Cid: m.failWithCid} +} diff --git a/pkg/retriever/parallelpeerretriever.go b/pkg/retriever/parallelpeerretriever.go index d5112222..826c4f79 100644 --- a/pkg/retriever/parallelpeerretriever.go +++ b/pkg/retriever/parallelpeerretriever.go @@ -17,8 +17,6 @@ import ( "github.com/multiformats/go-multicodec" ) -type GetStorageProviderTimeout func(peer peer.ID) time.Duration - // TransportProtocol implements the protocol-specific portions of a parallel- // peer retriever. It is responsible for communicating with individual peers // and also bears responsibility for some of the peer-selection logic. @@ -30,7 +28,6 @@ type TransportProtocol interface { ctx context.Context, retrieval *retrieval, shared *retrievalShared, - timeout time.Duration, candidate types.RetrievalCandidate, ) (*types.RetrievalStats, error) } @@ -140,10 +137,10 @@ func (retrieval *retrieval) RetrieveFromAsyncCandidates(asyncCandidates types.In finishAll <- struct{}{} }() - eventsCallback := func(evt types.RetrievalEvent) { + eventsCallback := func(peerID peer.ID, evt types.RetrievalEvent) { switch ret := evt.(type) { case events.FirstByteEvent: - retrieval.Session.RecordFirstByteTime(ret.ProviderId(), ret.Duration()) + retrieval.Session.RecordFirstByteTime(peerID, ret.Duration()) } retrieval.eventsCallback(evt) } @@ -227,22 +224,14 @@ func (retrieval *retrieval) runRetrievalCandidate( shared *retrievalShared, candidate types.RetrievalCandidate, ) { - timeout := retrieval.Session.GetStorageProviderTimeout(candidate.MinerPeer.ID) - var stats *types.RetrievalStats var retrievalErr error var done func() - shared.sendEvent(ctx, events.StartedRetrieval(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code())) - connectCtx := ctx - if timeout != 0 { - var timeoutFunc func() - connectCtx, timeoutFunc = retrieval.parallelPeerRetriever.Clock.WithDeadline(ctx, retrieval.parallelPeerRetriever.Clock.Now().Add(timeout)) - defer timeoutFunc() - } + shared.sendEvent(ctx, candidate.MinerPeer.ID, events.StartedRetrieval(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code())) // Setup in parallel - connectTime, err := retrieval.Protocol.Connect(connectCtx, retrieval, candidate) + connectTime, err := retrieval.Protocol.Connect(ctx, retrieval, candidate) if err != nil { // Exclude the case where the context was cancelled by the parent, which likely means that // another protocol has succeeded. @@ -252,10 +241,10 @@ func (retrieval *retrieval) runRetrievalCandidate( if err := retrieval.Session.RecordFailure(retrieval.request.RetrievalID, candidate.MinerPeer.ID); err != nil { logger.Errorf("Error recording retrieval failure on protocol %s: %v", retrieval.Protocol.Code().String(), err) } - shared.sendEvent(ctx, events.FailedRetrieval(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code(), retrievalErr.Error())) + shared.sendEvent(ctx, candidate.MinerPeer.ID, events.FailedRetrieval(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code(), retrievalErr.Error())) } } else { - shared.sendEvent(ctx, events.ConnectedToProvider(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code())) + shared.sendEvent(ctx, candidate.MinerPeer.ID, events.ConnectedToProvider(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code())) retrieval.Session.RecordConnectTime(candidate.MinerPeer.ID, connectTime) @@ -263,23 +252,19 @@ func (retrieval *retrieval) runRetrievalCandidate( done = shared.waitQueue.Wait(candidate.MinerPeer.ID) if shared.canSendResult() { // move on to retrieval - stats, retrievalErr = retrieval.Protocol.Retrieve(ctx, retrieval, shared, timeout, candidate) + stats, retrievalErr = retrieval.Protocol.Retrieve(ctx, retrieval, shared, candidate) if retrievalErr != nil { // Exclude the case where the context was cancelled by the parent, which likely // means that another protocol has succeeded. if !errors.Is(ctx.Err(), context.Canceled) { - msg := retrievalErr.Error() - if errors.Is(retrievalErr, ErrRetrievalTimedOut) { - msg = fmt.Sprintf("%s after %s", ErrRetrievalTimedOut.Error(), timeout) - } - shared.sendEvent(ctx, events.FailedRetrieval(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code(), msg)) + shared.sendEvent(ctx, candidate.MinerPeer.ID, events.FailedRetrieval(retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, retrieval.Protocol.Code(), retrievalErr.Error())) if err := retrieval.Session.RecordFailure(retrieval.request.RetrievalID, candidate.MinerPeer.ID); err != nil { logger.Errorf("Error recording retrieval failure for protocol %s: %v", retrieval.Protocol.Code().String(), err) } } } else { - shared.sendEvent(ctx, events.Success( + shared.sendEvent(ctx, candidate.MinerPeer.ID, events.Success( retrieval.parallelPeerRetriever.Clock.Now(), retrieval.request.RetrievalID, candidate, diff --git a/pkg/retriever/proposal.go b/pkg/retriever/proposal.go deleted file mode 100644 index 92427524..00000000 --- a/pkg/retriever/proposal.go +++ /dev/null @@ -1,46 +0,0 @@ -package retriever - -import ( - "sync/atomic" - "time" - - retrievaltypes "github.com/filecoin-project/go-retrieval-types" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" -) - -var dealIdGen = NewTimeCounter() - -// timeCounter is used to generate a monotonically increasing sequence. -// It starts at the current time, then increments on each call to next. -type TimeCounter struct { - counter uint64 -} - -func NewTimeCounter() *TimeCounter { - return &TimeCounter{counter: uint64(time.Now().UnixNano())} -} - -func (tc *TimeCounter) Next() uint64 { - counter := atomic.AddUint64(&tc.counter, 1) - return counter -} - -func RetrievalProposalForAsk(ask *retrievaltypes.QueryResponse, c cid.Cid, selector ipld.Node) (*retrievaltypes.DealProposal, error) { - params, err := retrievaltypes.NewParamsV1( - ask.MinPricePerByte, - ask.MaxPaymentInterval, - ask.MaxPaymentIntervalIncrease, - selector, - nil, - ask.UnsealPrice, - ) - if err != nil { - return nil, err - } - return &retrievaltypes.DealProposal{ - PayloadCID: c, - ID: retrievaltypes.DealID(dealIdGen.Next()), - Params: params, - }, nil -} diff --git a/pkg/retriever/proposal_test.go b/pkg/retriever/proposal_test.go deleted file mode 100644 index bb90260a..00000000 --- a/pkg/retriever/proposal_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package retriever - -import ( - "testing" - - retrievaltypes "github.com/filecoin-project/go-retrieval-types" - "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" - "github.com/stretchr/testify/require" -) - -func TestRetrievalProposalForAsk(t *testing.T) { - cid1 := cid.MustParse("bafkqaalb") - ask := &retrievaltypes.QueryResponse{ - MinPricePerByte: abi.NewTokenAmount(1), - MaxPaymentInterval: 2, - MaxPaymentIntervalIncrease: 3, - UnsealPrice: abi.NewTokenAmount(4), - } - proposal, err := RetrievalProposalForAsk(ask, cid1, selectorparse.CommonSelector_ExploreAllRecursively) - require.NoError(t, err) - require.NotZero(t, proposal.ID) - require.Equal(t, cid1, proposal.PayloadCID) - require.Same(t, selectorparse.CommonSelector_ExploreAllRecursively, proposal.Params.Selector.Node) - require.Nil(t, proposal.Params.PieceCID) - require.Equal(t, abi.NewTokenAmount(1), proposal.Params.PricePerByte) - require.Equal(t, uint64(2), proposal.Params.PaymentInterval) - require.Equal(t, uint64(3), proposal.Params.PaymentIntervalIncrease) - require.Equal(t, abi.NewTokenAmount(4), proposal.Params.UnsealPrice) - - proposal, err = RetrievalProposalForAsk(ask, cid1, selectorparse.CommonSelector_MatchPoint) - require.NoError(t, err) - require.NotZero(t, proposal.ID) - require.Equal(t, cid1, proposal.PayloadCID) - require.Same(t, selectorparse.CommonSelector_MatchPoint, proposal.Params.Selector.Node) - require.Nil(t, proposal.Params.PieceCID) - require.Equal(t, abi.NewTokenAmount(1), proposal.Params.PricePerByte) - require.Equal(t, uint64(2), proposal.Params.PaymentInterval) - require.Equal(t, uint64(3), proposal.Params.PaymentIntervalIncrease) - require.Equal(t, abi.NewTokenAmount(4), proposal.Params.UnsealPrice) -} diff --git a/pkg/retriever/protocolsplitter.go b/pkg/retriever/protocolsplitter.go deleted file mode 100644 index fdea28e6..00000000 --- a/pkg/retriever/protocolsplitter.go +++ /dev/null @@ -1,45 +0,0 @@ -package retriever - -import ( - "context" - - "github.com/filecoin-project/lassie/pkg/types" - "github.com/multiformats/go-multicodec" -) - -type ProtocolSplitter struct { - protocols []multicodec.Code -} - -var _ types.CandidateSplitter[multicodec.Code] = (*ProtocolSplitter)(nil) - -func NewProtocolSplitter(protocols []multicodec.Code) types.CandidateSplitter[multicodec.Code] { - return &ProtocolSplitter{protocols: protocols} -} - -func (ps *ProtocolSplitter) SplitRetrievalRequest(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.RetrievalSplitter[multicodec.Code] { - - return &retrievalProtocolSplitter{ps, request.GetSupportedProtocols(ps.protocols)} -} - -type retrievalProtocolSplitter struct { - *ProtocolSplitter - protocols []multicodec.Code -} - -func (rps *retrievalProtocolSplitter) SplitCandidates(candidates []types.RetrievalCandidate) (map[multicodec.Code][]types.RetrievalCandidate, error) { - protocolCandidates := make(map[multicodec.Code][]types.RetrievalCandidate, len(rps.protocols)) - for _, candidate := range candidates { - candidateProtocolsArr := candidate.Metadata.Protocols() - candidateProtocolsSet := make(map[multicodec.Code]struct{}) - for _, candidateProtocol := range candidateProtocolsArr { - candidateProtocolsSet[candidateProtocol] = struct{}{} - } - for _, protocol := range rps.protocols { - if _, ok := candidateProtocolsSet[protocol]; ok { - protocolCandidates[protocol] = append(protocolCandidates[protocol], candidate) - } - } - } - return protocolCandidates, nil -} diff --git a/pkg/retriever/resultcollector.go b/pkg/retriever/resultcollector.go index 6027ffeb..b727466e 100644 --- a/pkg/retriever/resultcollector.go +++ b/pkg/retriever/resultcollector.go @@ -3,7 +3,6 @@ package retriever import ( "context" - "github.com/filecoin-project/lassie/pkg/events" "github.com/filecoin-project/lassie/pkg/retriever/prioritywaitqueue" "github.com/filecoin-project/lassie/pkg/types" "github.com/libp2p/go-libp2p/core/peer" @@ -62,21 +61,20 @@ func (shared *retrievalShared) sendAllFinished(ctx context.Context) { } } -func (shared *retrievalShared) sendEvent(ctx context.Context, event events.EventWithProviderID) { - retrievalEvent := event.(types.RetrievalEvent) - shared.sendResult(ctx, retrievalResult{PeerID: event.ProviderId(), Event: &retrievalEvent}) +func (shared *retrievalShared) sendEvent(ctx context.Context, peerID peer.ID, event types.RetrievalEvent) { + shared.sendResult(ctx, retrievalResult{PeerID: peerID, Event: &event}) } // collectResults is responsible for receiving query errors, retrieval errors // and retrieval results and aggregating into an appropriate return of either // a complete RetrievalStats or an bundled multi-error -func collectResults(ctx context.Context, shared *retrievalShared, eventsCallback func(types.RetrievalEvent)) (*types.RetrievalStats, error) { +func collectResults(ctx context.Context, shared *retrievalShared, eventsCallback func(peer.ID, types.RetrievalEvent)) (*types.RetrievalStats, error) { var retrievalErrors error for { select { case result := <-shared.resultChan: if result.Event != nil { - eventsCallback(*result.Event) + eventsCallback(result.PeerID, *result.Event) break } if result.Err != nil { diff --git a/pkg/retriever/retriever.go b/pkg/retriever/retriever.go index c94a388a..2eaf107d 100644 --- a/pkg/retriever/retriever.go +++ b/pkg/retriever/retriever.go @@ -4,13 +4,14 @@ import ( "context" "errors" "fmt" + "net/http" "strings" "time" "github.com/dustin/go-humanize" "github.com/filecoin-project/go-clock" "github.com/filecoin-project/lassie/pkg/events" - "github.com/filecoin-project/lassie/pkg/retriever/combinators" + "github.com/filecoin-project/lassie/pkg/extractor" "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" @@ -36,7 +37,6 @@ var ( ) type Session interface { - GetStorageProviderTimeout(storageProviderId peer.ID) time.Duration FilterIndexerCandidate(candidate types.RetrievalCandidate) (bool, types.RetrievalCandidate) RegisterRetrieval(retrievalId types.RetrievalID, cid cid.Cid, selector datamodel.Node) bool @@ -53,11 +53,13 @@ type Session interface { type Retriever struct { // Assumed immutable during operation - executor types.Retriever - eventManager *events.EventManager - session Session - clock clock.Clock - protocols []multicodec.Code + executor types.Retriever + eventManager *events.EventManager + session Session + clock clock.Clock + protocols []multicodec.Code + candidateFinder types.CandidateFinder + candidateRetriever types.CandidateRetriever } type eventStats struct { @@ -68,39 +70,45 @@ func NewRetriever( ctx context.Context, session Session, candidateSource types.CandidateSource, - protocolRetrievers map[multicodec.Code]types.CandidateRetriever, + candidateRetriever types.CandidateRetriever, + protocol multicodec.Code, ) (*Retriever, error) { - return NewRetrieverWithClock(ctx, session, candidateSource, protocolRetrievers, clock.New()) + return NewRetrieverWithClock(ctx, session, candidateSource, candidateRetriever, protocol, clock.New()) } func NewRetrieverWithClock( ctx context.Context, session Session, candidateSource types.CandidateSource, - protocolRetrievers map[multicodec.Code]types.CandidateRetriever, + candidateRetriever types.CandidateRetriever, + protocol multicodec.Code, clock clock.Clock, ) (*Retriever, error) { retriever := &Retriever{ - eventManager: events.NewEventManager(ctx), - session: session, - clock: clock, - } - retriever.protocols = []multicodec.Code{} - for protocol := range protocolRetrievers { - retriever.protocols = append(retriever.protocols, protocol) - } - retriever.executor = combinators.RetrieverWithCandidateFinder{ - CandidateFinder: NewAssignableCandidateFinderWithClock(candidateSource, session.FilterIndexerCandidate, clock), - CandidateRetriever: combinators.SplitRetriever[multicodec.Code]{ - AsyncCandidateSplitter: combinators.NewAsyncCandidateSplitter(retriever.protocols, NewProtocolSplitter), - CandidateRetrievers: protocolRetrievers, - CoordinationKind: types.RaceCoordination, - }, + eventManager: events.NewEventManager(ctx), + session: session, + clock: clock, + protocols: []multicodec.Code{protocol}, + candidateFinder: NewAssignableCandidateFinderWithClock(candidateSource, session.FilterIndexerCandidate, clock), + candidateRetriever: candidateRetriever, } return retriever, nil } +func (retriever *Retriever) WrapWithHybrid(candidateSource types.CandidateSource, httpClient *http.Client, skipBlockVerification bool) { + base := &baseRetriever{retriever: retriever} + retriever.executor = NewHybridRetriever(base, candidateSource, httpClient, skipBlockVerification) +} + +type baseRetriever struct { + retriever *Retriever +} + +func (br *baseRetriever) Retrieve(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) (*types.RetrievalStats, error) { + return br.retriever.retrieveWithCandidates(ctx, request, events) +} + // Start will start the retriever events system func (retriever *Retriever) Start() { retriever.eventManager.Start() @@ -163,11 +171,14 @@ func (retriever *Retriever) Retrieve( // (retrievalStats!=nil) _and_ also an error return because there may be // multiple failures along the way, if we got a retrieval then we'll pretend // to our caller that there was no error - retrievalStats, err := retriever.executor.Retrieve( - ctx, - request, - onRetrievalEvent, - ) + var retrievalStats *types.RetrievalStats + if retriever.executor != nil { + // Use wrapped executor (e.g., HybridRetriever) + retrievalStats, err = retriever.executor.Retrieve(ctx, request, onRetrievalEvent) + } else { + // Direct retrieval using candidate finder and retriever + retrievalStats, err = retriever.retrieveWithCandidates(ctx, request, onRetrievalEvent) + } // Emit a Finished event denoting that the entire fetch has finished onRetrievalEvent(events.Finished(retriever.clock.Now(), request.RetrievalID, types.RetrievalCandidate{RootCid: request.Root})) @@ -177,21 +188,71 @@ func (retriever *Retriever) Retrieve( } // success + provider := retrievalStats.StorageProviderId.String() + if len(retrievalStats.Providers) > 0 { + provider = strings.Join(retrievalStats.Providers, ", ") + } logger.Infof( - "Successfully retrieved from miner %s for %s\n"+ + "Successfully retrieved from %s for %s\n"+ "\tDuration: %s\n"+ - "\tBytes Received: %s\n"+ - "\tTotal Payment: %s", - retrievalStats.StorageProviderId, + "\tBytes Received: %s", + provider, request.Root, retrievalStats.Duration, humanize.IBytes(retrievalStats.Size), - types.FIL(retrievalStats.TotalPayment), ) return retrievalStats, nil } +// retrieveWithCandidates performs retrieval by finding candidates and passing them +// to the candidate retriever. This is the simplified direct retrieval path. +func (retriever *Retriever) retrieveWithCandidates( + ctx context.Context, + request types.RetrievalRequest, + events func(types.RetrievalEvent), +) (*types.RetrievalStats, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + asyncCandidateRetrieval := retriever.candidateRetriever.Retrieve(ctx, request, events) + + asyncCandidates := make(chan []types.RetrievalCandidate) + outbound := types.OutboundAsyncCandidates(asyncCandidates) + inbound := types.InboundAsyncCandidates(asyncCandidates) + + findErr := make(chan error, 1) + resultChan := make(chan types.RetrievalResult, 1) + + go func(findErr chan<- error) { + defer close(findErr) + defer close(outbound) + err := retriever.candidateFinder.FindCandidates(ctx, request, events, func(candidates []types.RetrievalCandidate) { + _ = outbound.SendNext(ctx, candidates) + }) + findErr <- err + }(findErr) + + go func() { + stats, err := asyncCandidateRetrieval.RetrieveFromAsyncCandidates(inbound) + resultChan <- types.RetrievalResult{Stats: stats, Err: err} + }() + + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case err := <-findErr: + if err != nil { + return nil, err + } + findErr = nil + case result := <-resultChan: + return result.Stats, result.Err + } + } +} + // Implement RetrievalSubscriber func makeOnRetrievalEvent( ctx context.Context, @@ -230,8 +291,8 @@ func handleFailureEvent( ) { eventStats.failedCount++ logger.Warnf( - "Failed to retrieve from miner %s for %s: %s", - event.ProviderId(), + "Failed to retrieve from %s for %s: %s", + event.Endpoint(), event.RootCid(), event.ErrorMessage(), ) @@ -254,6 +315,35 @@ func handleCandidatesFilteredEvent( } } +// RetrieveAndExtract extracts content directly to disk using streaming extraction. +// Requires the retriever to be wrapped with HybridRetriever. +func (retriever *Retriever) RetrieveAndExtract( + ctx context.Context, + rootCid cid.Cid, + ext *extractor.Extractor, + eventsCallback func(types.RetrievalEvent), + onBlock func(int), +) (*types.RetrievalStats, error) { + if !retriever.eventManager.IsStarted() { + return nil, ErrRetrieverNotStarted + } + + hr, ok := retriever.executor.(*HybridRetriever) + if !ok { + return nil, errors.New("streaming extraction requires HybridRetriever") + } + + // wrap callback to also dispatch to subscribers + wrappedCallback := func(event types.RetrievalEvent) { + retriever.eventManager.DispatchEvent(event) + if eventsCallback != nil { + eventsCallback(event) + } + } + + return hr.RetrieveAndExtract(ctx, rootCid, ext, wrappedCallback, onBlock) +} + func logEvent(event types.RetrievalEvent) { kv := make([]interface{}, 0) logadd := func(kva ...interface{}) { @@ -275,7 +365,7 @@ func logEvent(event types.RetrievalEvent) { case events.EventWithCandidates: var cands = strings.Builder{} for i, c := range tevent.Candidates() { - cands.WriteString(c.MinerPeer.ID.String()) + cands.WriteString(c.Endpoint()) if i < len(tevent.Candidates())-1 { cands.WriteString(", ") } diff --git a/pkg/retriever/retriever_test.go b/pkg/retriever/retriever_test.go deleted file mode 100644 index 9e07eb2e..00000000 --- a/pkg/retriever/retriever_test.go +++ /dev/null @@ -1,1084 +0,0 @@ -package retriever - -import ( - "context" - "errors" - "math" - "testing" - "time" - - "github.com/filecoin-project/go-clock" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lassie/pkg/events" - "github.com/filecoin-project/lassie/pkg/internal/testutil" - "github.com/filecoin-project/lassie/pkg/session" - "github.com/filecoin-project/lassie/pkg/types" - "github.com/google/uuid" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/linking" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" - trustlessutils "github.com/ipld/go-trustless-utils" - "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multicodec" - "github.com/stretchr/testify/require" -) - -func TestRetrieverStart(t *testing.T) { - candidateSource := &testutil.MockCandidateSource{} - client := &testutil.MockClient{} - session := session.NewSession(nil, true) - gsretriever := NewGraphsyncRetriever(session, client) - ret, err := NewRetriever(context.Background(), session, candidateSource, map[multicodec.Code]types.CandidateRetriever{ - multicodec.TransportGraphsyncFilecoinv1: gsretriever, - }) - require.NoError(t, err) - - // --- run --- - result, err := ret.Retrieve(context.Background(), types.RetrievalRequest{ - LinkSystem: cidlink.DefaultLinkSystem(), - RetrievalID: types.RetrievalID(uuid.New()), - Request: trustlessutils.Request{Root: cid.MustParse("bafkqaalb")}, - }, func(types.RetrievalEvent) {}) - require.ErrorIs(t, err, ErrRetrieverNotStarted) - require.Nil(t, result) -} - -func TestRetriever(t *testing.T) { - rid := types.RetrievalID(uuid.New()) - cid1 := cid.MustParse("bafkqaalb") - peerA := peer.ID("A") - peerB := peer.ID("B") - initialPause := time.Millisecond * 5 - blacklistedPeer := peer.ID("blacklisted") - startTime := time.Now().Add(time.Hour) - tc := []struct { - name string - setup func(*testutil.MockSession) - candidates []types.RetrievalCandidate - path string - dups bool - scope trustlessutils.DagScope - returns_connected map[string]testutil.DelayedConnectReturn - returns_retrievals map[string]testutil.DelayedClientReturn - cancelAfter time.Duration - successfulPeer peer.ID - err error - expectedSequence []testutil.ExpectedActionsAtTime - }{ - { - name: "single candidate and successful retrieval", - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Millisecond * 20}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerA): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 1, - Blocks: 2, - Duration: 3 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - }, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 20 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(20*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerA, Duration: 20 * time.Millisecond}, - }, - }, - { - AfterStart: 20*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerA}, - }, - { - AfterStart: 25*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.Accepted(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.FirstByte(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 1, 2, 3*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(25*time.Millisecond+initialPause), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerA, Duration: 5 * time.Millisecond}, - {Type: testutil.SessionMetric_Success, Provider: peerA, Value: math.Trunc(1.0 / float64((3 * time.Second).Milliseconds()))}, - }, - }, - }, - }, - { - name: "single candidate and successful retrieval, w/ path & dups & scope", - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - path: "some/path/to/request", - dups: true, - scope: trustlessutils.DagScopeBlock, - returns_connected: map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Millisecond * 20}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerA): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 1, - Blocks: 2, - Duration: 3 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - }, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "/some/path/to/request?dag-scope=block&dups=y", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 20 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(20*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerA, Duration: 20 * time.Millisecond}, - }, - }, - { - AfterStart: 20*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerA}, - }, - { - AfterStart: 25*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.Accepted(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.FirstByte(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(25*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 1, 2, 3*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(25*time.Millisecond+initialPause), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerA, Duration: 5 * time.Millisecond}, - {Type: testutil.SessionMetric_Success, Provider: peerA, Value: math.Trunc(1.0 / float64((3 * time.Second).Milliseconds()))}, - }, - }, - }, - }, - { - name: "two candidates, fast one wins", - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - {MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Second}, - string(peerB): {Err: nil, Delay: time.Millisecond * 5}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerB): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerB, - Size: 10, - Blocks: 11, - Duration: 12 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - }, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA, peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(5*time.Millisecond), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerB, Duration: 5 * time.Millisecond}, - }, - }, - { - AfterStart: 5*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerB}, - }, - { - AfterStart: 10*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.Accepted(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.FirstByte(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 10, 11, 12*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(10*time.Millisecond+initialPause), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerB, Duration: 5 * time.Millisecond}, - {Type: testutil.SessionMetric_Success, Provider: peerB, Value: math.Trunc(10.0 / float64((12 * time.Second).Milliseconds()))}, - }, - }, - }, - }, - { - name: "blacklisted candidate", - setup: func(ms *testutil.MockSession) { - ms.SetBlockList(map[peer.ID]bool{blacklistedPeer: true}) - }, - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: blacklistedPeer}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{ - // fastest is blacklisted, shouldn't even touch it - string(blacklistedPeer): {Err: nil, Delay: time.Millisecond * 5}, - string(peerA): {Err: nil, Delay: time.Millisecond * 50}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerA): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 1, - Blocks: 2, - Duration: 3 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - }, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: blacklistedPeer}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(blacklistedPeer, nil, cid1), types.NewRetrievalCandidate(peerA, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 50 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(50*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerA, Duration: 50 * time.Millisecond}, - }, - }, - { - AfterStart: 50*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerA}, - }, - { - AfterStart: 55*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.Accepted(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.FirstByte(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 1, 2, 3*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(55*time.Millisecond+initialPause), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerA, Duration: 5 * time.Millisecond}, - {Type: testutil.SessionMetric_Success, Provider: peerA, Value: math.Trunc(1.0 / float64((3 * time.Second).Milliseconds()))}, - }, - }, - }, - }, - - { - name: "two candidates, fast one fails connect, slow wins", - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - {MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: errors.New("blip"), Delay: time.Millisecond * 5}, - string(peerB): {Err: nil, Delay: time.Millisecond * 50}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerB): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 1, - Blocks: 2, - Duration: 3 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - }, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA, peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.FailedRetrieval(startTime.Add(5*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, "unable to connect to provider: blip"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerA}, - }, - }, - { - AfterStart: 50 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(50*time.Millisecond), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerB, Duration: 50 * time.Millisecond}, - }, - }, - { - AfterStart: 50*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerB}, - }, - { - AfterStart: 55*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.Accepted(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.FirstByte(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(55*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 1, 2, 3*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(55*time.Millisecond+initialPause), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerB, Duration: 5 * time.Millisecond}, - {Type: testutil.SessionMetric_Success, Provider: peerB, Value: math.Trunc(1.0 / float64((3 * time.Second).Milliseconds()))}, - }, - }, - }, - }, - - { - name: "two candidates, fast one fails retrieval, slow wins", - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - {MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Millisecond * 500}, - string(peerB): {Err: nil, Delay: time.Millisecond * 5}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerA): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 10, - Blocks: 20, - Duration: 30 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - string(peerB): {ResultStats: nil, ResultErr: errors.New("bork!"), Delay: time.Millisecond * 5}, - }, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA, peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(5*time.Millisecond), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerB, Duration: 5 * time.Millisecond}, - }, - }, - { - AfterStart: 5*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerB}, - }, - { - AfterStart: 10*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.FailedRetrieval(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, "retrieval failed: bork!"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerB}, - }, - }, - { - AfterStart: 500 * time.Millisecond, - ReceivedRetrievals: []peer.ID{peerA}, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(500*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerA, Duration: 500 * time.Millisecond}, - }, - }, - { - AfterStart: 505 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(505*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.Accepted(startTime.Add(505*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.FirstByte(startTime.Add(505*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(505*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(505*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 10, 20, 30*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(505*time.Millisecond), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerA, Duration: 5 * time.Millisecond}, - {Type: testutil.SessionMetric_Success, Provider: peerA, Value: math.Trunc(10.0 / float64((30 * time.Second).Milliseconds()))}, - }, - }, - }, - }, - - { - name: "two candidates, first times out retrieval", - setup: func(ms *testutil.MockSession) { - ms.SetProviderTimeout(time.Millisecond * 200) - }, - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - {MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Millisecond}, - string(peerB): {Err: nil, Delay: time.Millisecond * 100}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerA): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 10, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Second * 2}, - string(peerB): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerB, - Size: 20, - Blocks: 30, - Duration: 40 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond}, - }, - successfulPeer: peerB, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA, peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 1 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(1*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerA, Duration: time.Millisecond}, - }, - }, - { - AfterStart: 1*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerA}, - }, - { - AfterStart: 100 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(100*time.Millisecond), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerB, Duration: 100 * time.Millisecond}, - }, - }, - { - AfterStart: 201*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.FailedRetrieval(startTime.Add(201*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, "retrieval timed out 200ms"), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Failure, Provider: peerA}, - }, - }, - { - AfterStart: 202*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(202*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.Accepted(startTime.Add(202*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.FirstByte(startTime.Add(202*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 1*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(202*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(202*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 20, 30, 40*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(202*time.Millisecond+initialPause), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_FirstByte, Provider: peerB, Duration: time.Millisecond}, - {Type: testutil.SessionMetric_Success, Provider: peerB, Value: math.Trunc(20.0 / float64((34 * time.Second).Milliseconds()))}, - }, - }, - }, - }, - - { - name: "no candidates", - setup: func(ms *testutil.MockSession) { - ms.SetProviderTimeout(time.Millisecond * 100) - }, - candidates: []types.RetrievalCandidate{}, - returns_connected: map[string]testutil.DelayedConnectReturn{}, - returns_retrievals: map[string]testutil.DelayedClientReturn{}, - successfulPeer: peer.ID(""), - err: ErrNoCandidates, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.Failed(startTime, rid, types.RetrievalCandidate{RootCid: cid1}, "no candidates"), - events.Finished(startTime, rid, types.RetrievalCandidate{RootCid: cid1}), - }, - }, - }, - }, - { - name: "no acceptable candidates", - setup: func(ms *testutil.MockSession) { - ms.SetProviderTimeout(time.Millisecond * 100) - ms.SetBlockList(map[peer.ID]bool{blacklistedPeer: true}) - }, - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: blacklistedPeer}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{}, - returns_retrievals: map[string]testutil.DelayedClientReturn{}, - successfulPeer: peer.ID(""), - err: ErrNoCandidates, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: blacklistedPeer}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(blacklistedPeer, nil, cid1)}), - events.Failed(startTime, rid, types.RetrievalCandidate{RootCid: cid1}, "no candidates"), - events.Finished(startTime, rid, types.RetrievalCandidate{RootCid: cid1}), - }, - }, - }, - }, - { - // testing context cancellation, but this doesn't quite test all the - // way into the protocol (graphsync) retriever itself since the - // Retriever sits in between and has multiple paths to handling - // context cancellation. A leaky goroutine can still occur deeper - // in. A test with the same name directly against the - // GraphsyncRetriever gets more accurately in. - name: "context cancelled before completed", - candidates: []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - {MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - returns_connected: map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Millisecond * 1}, - string(peerB): {Err: nil, Delay: time.Millisecond * 5}, - }, - returns_retrievals: map[string]testutil.DelayedClientReturn{ - string(peerA): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 10, - Blocks: 11, - Duration: 12 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Second * 1}, - string(peerB): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerB, - Size: 10, - Blocks: 11, - Duration: 12 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Second * 1}, - }, - cancelAfter: time.Millisecond * 100, - err: context.Canceled, - expectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA, peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(1*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerA, Duration: 1 * time.Millisecond}, - }, - }, - { - AfterStart: 5 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(5*time.Millisecond), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - ExpectedMetrics: []testutil.SessionMetric{ - {Type: testutil.SessionMetric_Connect, Provider: peerB, Duration: 5 * time.Millisecond}, - }, - }, - { - AfterStart: 5*time.Millisecond + initialPause, - }, - { - AfterStart: 100 * time.Millisecond, - // context cancellation here should cause the retrieval to fail and not emit more events - }, - { - AfterStart: 1*time.Millisecond + 1*time.Second + initialPause, - }, - }, - }, - } - - ctx := context.Background() - for _, tc := range tc { - tc := tc - t.Run(tc.name, func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - retCtx, retCancel := context.WithCancel(ctx) - defer retCancel() - - clock := clock.NewMock() - clock.Set(startTime) - // --- setup --- - candidateSource := testutil.NewMockCandidateSource(nil, map[cid.Cid][]types.RetrievalCandidate{cid1: tc.candidates}) - client := testutil.NewMockClient(tc.returns_connected, tc.returns_retrievals, clock) - session := testutil.NewMockSession(ctx) - if tc.setup != nil { - tc.setup(session) - } - gsretriever := NewGraphsyncRetrieverWithConfig(session, client, clock, initialPause, true) - - // --- create --- - ret, err := NewRetrieverWithClock(context.Background(), session, candidateSource, map[multicodec.Code]types.CandidateRetriever{ - multicodec.TransportGraphsyncFilecoinv1: gsretriever, - }, clock) - require.NoError(t, err) - - // --- start --- - ret.Start() - - // --- retrieve --- - require.NoError(t, err) - results := testutil.RetrievalVerifier{ - ExpectedSequence: tc.expectedSequence, - }.RunWithVerification( - ctx, - t, - clock, - client, - candidateSource, - session, - retCancel, - tc.cancelAfter, - []testutil.RunRetrieval{func(cb func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - return ret.Retrieve(retCtx, types.RetrievalRequest{ - LinkSystem: cidlink.DefaultLinkSystem(), - RetrievalID: rid, - Request: trustlessutils.Request{ - Root: cid1, - Path: tc.path, - Scope: tc.scope, - Duplicates: tc.dups, - }, - }, cb) - }}, - ) - - // --- stop --- - ret.Stop() - - // -- verify -- - require.Len(t, results, 1) - result, err := results[0].Stats, results[0].Err - if tc.err == nil { - require.NoError(t, err) - successfulPeer := string(tc.successfulPeer) - if successfulPeer == "" { - for p, retrievalReturns := range tc.returns_retrievals { - if retrievalReturns.ResultStats != nil { - successfulPeer = p - } - } - } - require.Equal(t, client.GetRetrievalReturns()[successfulPeer].ResultStats, result) - } else { - require.ErrorIs(t, tc.err, err) - } - }) - } -} - -func TestLinkSystemPerRequest(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - startTime := time.Now().Add(5 * time.Hour) - clock := clock.NewMock() - clock.Set(startTime) - initialPause := time.Millisecond * 2 - rid := types.RetrievalID(uuid.New()) - cid1 := cid.MustParse("bafkqaalb") - peerA := peer.ID("A") - peerB := peer.ID("B") - - candidates := []types.RetrievalCandidate{ - {MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - {MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - } - returnsConnected := map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Millisecond * 5}, - string(peerB): {Err: nil, Delay: time.Millisecond * 500}, - } - returnsRetrievals := map[string]testutil.DelayedClientReturn{ - string(peerA): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerA, - Size: 1, - Blocks: 2, - Duration: 3 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - string(peerB): {ResultStats: &types.RetrievalStats{ - StorageProviderId: peerB, - Size: 10, - Blocks: 11, - Duration: 12 * time.Second, - TotalPayment: big.Zero(), - RootCid: cid1, - AskPrice: abi.NewTokenAmount(0), - }, Delay: time.Millisecond * 5}, - } - - candidateSource := testutil.NewMockCandidateSource(nil, map[cid.Cid][]types.RetrievalCandidate{cid1: candidates}) - client := testutil.NewMockClient(returnsConnected, returnsRetrievals, clock) - session := session.NewSession(nil, true) - gsretriever := NewGraphsyncRetrieverWithConfig(session, client, clock, initialPause, true) - - // --- create --- - ret, err := NewRetrieverWithClock(context.Background(), session, candidateSource, map[multicodec.Code]types.CandidateRetriever{ - multicodec.TransportGraphsyncFilecoinv1: gsretriever, - }, clock) - require.NoError(t, err) - - // --- start --- - ret.Start() - - // --- retrieve --- - lsA := cidlink.DefaultLinkSystem() - lsA.NodeReifier = func(lc linking.LinkContext, n datamodel.Node, ls *linking.LinkSystem) (datamodel.Node, error) { - return basicnode.NewString("linkSystem A"), nil - } - lsB := cidlink.DefaultLinkSystem() - lsB.NodeReifier = func(lc linking.LinkContext, n datamodel.Node, ls *linking.LinkSystem) (datamodel.Node, error) { - return basicnode.NewString("linkSystem B"), nil - } - results := testutil.RetrievalVerifier{ - ExpectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA, peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime, rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime, rid, cid1), - events.CandidatesFound(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.CandidatesFiltered(startTime, rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime, rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(5*time.Millisecond), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerA}, - }, - { - AfterStart: 10*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.Accepted(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1)), - events.FirstByte(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), 1, 2, 3*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add(10*time.Millisecond+initialPause), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - }, - }, - }.RunWithVerification(ctx, t, clock, client, candidateSource, nil, nil, 0, []testutil.RunRetrieval{ - func(cb func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - return ret.Retrieve(context.Background(), types.RetrievalRequest{ - LinkSystem: lsA, - RetrievalID: rid, - Request: trustlessutils.Request{Root: cid1}, - }, cb) - }, - }) - require.Len(t, results, 1) - result, err := results[0].Stats, results[0].Err - require.NoError(t, err) - require.Equal(t, returnsRetrievals[string(peerA)].ResultStats, result) - - // switch them around - returnsConnected = map[string]testutil.DelayedConnectReturn{ - string(peerA): {Err: nil, Delay: time.Millisecond * 500}, - string(peerB): {Err: nil, Delay: time.Millisecond * 5}, - } - client.SetConnectReturns(returnsConnected) - - // --- retrieve --- - results = testutil.RetrievalVerifier{ - ExpectedSequence: []testutil.ExpectedActionsAtTime{ - { - AfterStart: 0, - CandidatesDiscovered: []testutil.DiscoveredCandidate{ - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerA}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - { - Cid: cid1, - Candidate: types.RetrievalCandidate{MinerPeer: peer.AddrInfo{ID: peerB}, RootCid: cid1, Metadata: metadata.Default.New(&metadata.GraphsyncFilecoinV1{})}, - }, - }, - ReceivedConnections: []peer.ID{peerA, peerB}, - ExpectedEvents: []types.RetrievalEvent{ - events.StartedFetch(startTime.Add(10*time.Millisecond+initialPause), rid, cid1, "?dag-scope=all&dups=n", multicodec.TransportGraphsyncFilecoinv1), - events.StartedFindingCandidates(startTime.Add(10*time.Millisecond+initialPause), rid, cid1), - events.CandidatesFound(startTime.Add(10*time.Millisecond+initialPause), rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.CandidatesFiltered(startTime.Add(10*time.Millisecond+initialPause), rid, cid1, []types.RetrievalCandidate{types.NewRetrievalCandidate(peerA, nil, cid1), types.NewRetrievalCandidate(peerB, nil, cid1)}), - events.StartedRetrieval(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerA, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - events.StartedRetrieval(startTime.Add(10*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5 * time.Millisecond, - ExpectedEvents: []types.RetrievalEvent{ - events.ConnectedToProvider(startTime.Add(15*time.Millisecond+initialPause), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1), - }, - }, - { - AfterStart: 5*time.Millisecond + initialPause, - ReceivedRetrievals: []peer.ID{peerB}, - }, - { - AfterStart: 10*time.Millisecond + initialPause, - ExpectedEvents: []types.RetrievalEvent{ - events.Proposed(startTime.Add((10*time.Millisecond+initialPause)*2), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.Accepted(startTime.Add((10*time.Millisecond+initialPause)*2), rid, types.NewRetrievalCandidate(peerB, nil, cid1)), - events.FirstByte(startTime.Add((10*time.Millisecond+initialPause)*2), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 5*time.Millisecond, multicodec.TransportGraphsyncFilecoinv1), - events.BlockReceived(startTime.Add((10*time.Millisecond+initialPause)*2), rid, types.NewRetrievalCandidate(peerB, nil, cid1), multicodec.TransportGraphsyncFilecoinv1, 100), - events.Success(startTime.Add((10*time.Millisecond+initialPause)*2), rid, types.NewRetrievalCandidate(peerB, nil, cid1), 10, 11, 12*time.Second, multicodec.TransportGraphsyncFilecoinv1), - events.Finished(startTime.Add((10*time.Millisecond+initialPause)*2), rid, types.RetrievalCandidate{RootCid: cid1}), - }, - }, - }, - }.RunWithVerification(ctx, t, clock, client, candidateSource, nil, nil, 0, []testutil.RunRetrieval{ - func(cb func(types.RetrievalEvent)) (*types.RetrievalStats, error) { - return ret.Retrieve(context.Background(), types.RetrievalRequest{ - LinkSystem: lsB, - RetrievalID: rid, - Request: trustlessutils.Request{Root: cid1}, - }, cb) - }, - }) - require.Len(t, results, 1) - result, err = results[0].Stats, results[0].Err - - require.NoError(t, err) - require.Equal(t, returnsRetrievals[string(peerB)].ResultStats, result) - - // --- stop --- - ret.Stop() - - // --- verify --- - // two different linksystems for the different calls, in the order that we - // supplied them in our call to Retrieve() - require.Len(t, client.GetReceivedLinkSystems(), 2) - nd, err := client.GetReceivedLinkSystems()[0].NodeReifier(linking.LinkContext{}, nil, nil) - require.NoError(t, err) - str, err := nd.AsString() - require.NoError(t, err) - require.Equal(t, "linkSystem A", str) - nd, err = client.GetReceivedLinkSystems()[1].NodeReifier(linking.LinkContext{}, nil, nil) - require.NoError(t, err) - str, err = nd.AsString() - require.NoError(t, err) - require.Equal(t, "linkSystem B", str) -} diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index ffcb6547..c9ebe7bb 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -7,10 +7,8 @@ import ( "net/http" "net/url" "strconv" - "strings" "github.com/filecoin-project/lassie/pkg/build" - "github.com/filecoin-project/lassie/pkg/heyfil" "github.com/filecoin-project/lassie/pkg/retriever" "github.com/filecoin-project/lassie/pkg/storage" "github.com/filecoin-project/lassie/pkg/types" @@ -72,14 +70,6 @@ func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.Response request.LinkSystem.SetWriteStorage(carStore) request.LinkSystem.SetReadStorage(carStore) - // setup preload storage for bitswap, the temporary CAR store can set up a - // separate preload space in its storage - request.PreloadLinkSystem = cidlink.DefaultLinkSystem() - preloadStore := carStore.PreloadStore() - request.PreloadLinkSystem.SetReadStorage(preloadStore) - request.PreloadLinkSystem.SetWriteStorage(preloadStore) - request.PreloadLinkSystem.TrustedStorage = true - // bytesWritten will be closed once we've started writing CAR content to // the response writer. Once closed, no other content should be written. bytesWritten := make(chan struct{}, 1) @@ -188,6 +178,7 @@ func decodeRequest(res http.ResponseWriter, req *http.Request, unescapedPath str } if !accept.IsCar() { errorResponse(res, statusLogger, http.StatusNotAcceptable, fmt.Errorf("invalid Accept header or format parameter; unsupported %q", req.Header.Get("Accept"))) + return false, trustlessutils.Request{} } dagScope, err := trustlesshttp.ParseScope(req) @@ -298,14 +289,7 @@ func parseProtocols(req *http.Request) ([]multicodec.Code, error) { func parseProviders(req *http.Request) ([]types.Provider, error) { if req.URL.Query().Has("providers") { - // in case we have been given filecoin actor addresses we can look them up - // with heyfil and translate to full multiaddrs, otherwise this is a - // pass-through - trans, err := heyfil.Heyfil{TranslateFaddr: true}.TranslateAll(strings.Split(req.URL.Query().Get("providers"), ",")) - if err != nil { - return nil, err - } - providers, err := types.ParseProviderStrings(strings.Join(trans, ",")) + providers, err := types.ParseProviderStrings(req.URL.Query().Get("providers")) if err != nil { return nil, errors.New("invalid providers parameter") } diff --git a/pkg/server/http/server.go b/pkg/server/http/server.go index 318ebfad..f389dd73 100644 --- a/pkg/server/http/server.go +++ b/pkg/server/http/server.go @@ -16,10 +16,12 @@ var logger = log.Logger("lassie/httpserver") // HttpServer is a Lassie server for fetching data from the network via HTTP type HttpServer struct { - cancel context.CancelFunc - ctx context.Context - listener net.Listener - server *http.Server + cancel context.CancelFunc + ctx context.Context + listener net.Listener + server *http.Server + pprofListener net.Listener + pprofServer *http.Server } type HttpServerConfig struct { @@ -41,7 +43,7 @@ func saveConnInCTX(ctx context.Context, c net.Conn) context.Context { } // NewHttpServer creates a new HttpServer -func NewHttpServer(ctx context.Context, lassie *lassie.Lassie, cfg HttpServerConfig) (*HttpServer, error) { +func NewHttpServer(ctx context.Context, s *lassie.Lassie, cfg HttpServerConfig) (*HttpServer, error) { addr := fmt.Sprintf("%s:%d", cfg.Address, cfg.Port) listener, err := net.Listen("tcp", addr) // assigns a port if port is 0 if err != nil { @@ -73,14 +75,22 @@ func NewHttpServer(ctx context.Context, lassie *lassie.Lassie, cfg HttpServerCon } // Routes - mux.HandleFunc("/ipfs/", IpfsHandler(lassie, cfg)) + mux.HandleFunc("/ipfs/", IpfsHandler(s, cfg)) - // Handle pprof endpoints - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + // Setup pprof on localhost-only listener for security + pprofListener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + listener.Close() + return nil, fmt.Errorf("failed to create pprof listener: %w", err) + } + pprofMux := http.NewServeMux() + pprofMux.HandleFunc("/debug/pprof/", pprof.Index) + pprofMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + pprofMux.HandleFunc("/debug/pprof/profile", pprof.Profile) + pprofMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + pprofMux.HandleFunc("/debug/pprof/trace", pprof.Trace) + httpServer.pprofListener = pprofListener + httpServer.pprofServer = &http.Server{Handler: pprofMux} return httpServer, nil } @@ -93,6 +103,15 @@ func (s HttpServer) Addr() string { // Start starts the http server, returning an error if the server failed to start func (s *HttpServer) Start() error { logger.Infow("starting http server", "listen_addr", s.listener.Addr()) + logger.Infow("pprof available at", "pprof_addr", s.pprofListener.Addr()) + + // Start pprof server in background + go func() { + if err := s.pprofServer.Serve(s.pprofListener); err != nil && err != http.ErrServerClosed { + logger.Errorw("pprof server error", "err", err) + } + }() + err := s.server.Serve(s.listener) if err != http.ErrServerClosed { logger.Errorw("failed to start http server", "err", err) @@ -105,6 +124,7 @@ func (s *HttpServer) Start() error { func (s *HttpServer) Close() error { logger.Info("closing http server") s.cancel() + s.pprofServer.Shutdown(context.Background()) return s.server.Shutdown(context.Background()) } diff --git a/pkg/server/http/servertimingssubscriber.go b/pkg/server/http/servertimingssubscriber.go index 934f6f77..7593b621 100644 --- a/pkg/server/http/servertimingssubscriber.go +++ b/pkg/server/http/servertimingssubscriber.go @@ -102,7 +102,7 @@ func servertimingsSubscriber(req *http.Request, bytesWritten chan struct{}) type case events.FinishedEvent: default: - if event, ok := re.(events.EventWithProviderID); ok { + if event, ok := re.(events.EventWithEndpoint); ok { name := fmt.Sprintf("retrieval-%s", events.Identifier(event)) if retrievalMetric, ok := retrievalMetricMap[name]; ok { retrievalMetric.Extra[string(re.Code())] = fmt.Sprintf("%d", re.Time().Sub(retrievalTimingMap[name])) diff --git a/pkg/session/config.go b/pkg/session/config.go index 090aa423..c270a537 100644 --- a/pkg/session/config.go +++ b/pkg/session/config.go @@ -1,8 +1,7 @@ package session import ( - "math/rand" - "time" + "math/rand/v2" "github.com/libp2p/go-libp2p/core/peer" ) @@ -10,7 +9,6 @@ import ( type Random interface{ Float64() float64 } type ProviderConfig struct { - RetrievalTimeout time.Duration MaxConcurrentRetrievals uint } @@ -73,14 +71,6 @@ type Config struct { // recent success metric. SuccessAlpha float64 - // GraphsyncVerifiedDealWeight is the scoring weight applied when a - // graphsync candidate has a VerifiedDeal metadata. The weight is a - // multiplier of the base value of `1.0` when VerifiedDeal is found. - GraphsyncVerifiedDealWeight float64 - // GraphsyncFastRetrievalWeight is the scoring weight applied when a - // graphsync candidate has a FastRetrieval metadata. The weight is a - // multiplier of the base value of `1.0` when FastRetrieval is found. - GraphsyncFastRetrievalWeight float64 // ConnectTimeWeight is the scoring weight applied to the connect time // exponential moving average for the candidate at time of scoring. The // weight is a multiplier the base value, which should be a normalised @@ -107,19 +97,17 @@ type Config struct { // DefaultConfig returns a default config with usable alpha and weight values. func DefaultConfig() *Config { return &Config{ - ConnectTimeAlpha: 0.5, - OverallConnectTimeAlpha: 0.8, - FirstByteTimeAlpha: 0.5, - OverallFirstByteTimeAlpha: 0.8, - BandwidthAlpha: 0.5, - OverallBandwidthAlpha: 0.8, - SuccessAlpha: 0.5, - GraphsyncVerifiedDealWeight: 3.0, - GraphsyncFastRetrievalWeight: 2.0, - ConnectTimeWeight: 1.0, - FirstByteTimeWeight: 1.0, - BandwidthWeight: 0.5, - SuccessWeight: 1.0, + ConnectTimeAlpha: 0.5, + OverallConnectTimeAlpha: 0.8, + FirstByteTimeAlpha: 0.5, + OverallFirstByteTimeAlpha: 0.8, + BandwidthAlpha: 0.5, + OverallBandwidthAlpha: 0.8, + SuccessAlpha: 0.5, + ConnectTimeWeight: 1.0, + FirstByteTimeWeight: 1.0, + BandwidthWeight: 0.5, + SuccessWeight: 1.0, } } @@ -184,18 +172,6 @@ func (cfg Config) WithoutRandomness() *Config { return &cfg } -// WithGraphsyncVerifiedDealWeight sets the verified deal weight. -func (cfg Config) WithGraphsyncVerifiedDealWeight(weight float64) *Config { - cfg.GraphsyncVerifiedDealWeight = weight - return &cfg -} - -// WithGraphsyncFastRetrievalWeight sets the fast retrieval weight. -func (cfg Config) WithGraphsyncFastRetrievalWeight(weight float64) *Config { - cfg.GraphsyncFastRetrievalWeight = weight - return &cfg -} - // WithConnectTimeWeight sets the connect time weight. func (cfg Config) WithConnectTimeWeight(weight float64) *Config { cfg.ConnectTimeWeight = weight @@ -230,9 +206,6 @@ func (cfg *Config) getProviderConfig(peer peer.ID) ProviderConfig { if individual.MaxConcurrentRetrievals != 0 { minerCfg.MaxConcurrentRetrievals = individual.MaxConcurrentRetrievals } - if individual.RetrievalTimeout != 0 { - minerCfg.RetrievalTimeout = individual.RetrievalTimeout - } } return minerCfg } diff --git a/pkg/session/session.go b/pkg/session/session.go index 9997e0f0..aece9710 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -1,8 +1,6 @@ package session import ( - "time" - "github.com/filecoin-project/lassie/pkg/types" "github.com/ipni/go-libipni/metadata" "github.com/libp2p/go-libp2p/core/peer" @@ -39,12 +37,6 @@ func NewSession(config *Config, withState bool) *Session { return &Session{state, config} } -// GetStorageProviderTimeout returns the per-retrieval timeout from the -// RetrievalTimeout configuration option. -func (session *Session) GetStorageProviderTimeout(storageProviderId peer.ID) time.Duration { - return session.config.getProviderConfig(storageProviderId).RetrievalTimeout -} - // FilterIndexerCandidate filters out protocols that are not acceptable for // the given candidate. It returns a bool indicating whether the candidate // should be considered at all, and a new candidate with the filtered @@ -90,10 +82,6 @@ func (session *Session) isAcceptableCandidate(storageProviderId peer.ID) bool { } func (session *Session) isAcceptableCandidateForProtocol(storageProviderId peer.ID, protocol multicodec.Code) bool { - if protocol == multicodec.TransportBitswap { - return true - } - // check if we are currently retrieving from the candidate with its maximum // concurrency minerConfig := session.config.getProviderConfig(storageProviderId) diff --git a/pkg/session/state.go b/pkg/session/state.go index 0b78ebac..70f258eb 100644 --- a/pkg/session/state.go +++ b/pkg/session/state.go @@ -318,8 +318,7 @@ func (spt *SessionState) ChooseNextProvider(peers []peer.ID, mda []metadata.Prot ind := make([]int, len(peers)) // index into peers that we can sort without changing peers for ii, p := range peers { ind[ii] = ii - gsmd, _ := mda[ii].(*metadata.GraphsyncFilecoinV1) - scores[ii] = spt.scoreProvider(p, gsmd) + scores[ii] = spt.scoreProvider(p) tot += scores[ii] } // sort so that the non-random selection choose the first (best) @@ -347,21 +346,9 @@ func (spt *SessionState) ChooseNextProvider(peers []peer.ID, mda []metadata.Prot // scoreProvider returns a score for a given provider, higher is better. // This method assumes the caller holds lock. -func (spt *SessionState) scoreProvider(id peer.ID, md *metadata.GraphsyncFilecoinV1) float64 { +// HTTP-only: scoring based on collected metrics (connect time, bandwidth, success rate) +func (spt *SessionState) scoreProvider(id peer.ID) float64 { var score float64 - // var v, f bool - - // graphsync metadata weighting - if md != nil { - if md.VerifiedDeal { - score += spt.config.GraphsyncVerifiedDealWeight - // v = true - } - if md.FastRetrieval { - score += spt.config.GraphsyncFastRetrievalWeight - // f = true - } - } // collected metrics scoring sp := spt.spm[id] diff --git a/pkg/session/state_test.go b/pkg/session/state_test.go deleted file mode 100644 index 81a201a7..00000000 --- a/pkg/session/state_test.go +++ /dev/null @@ -1,587 +0,0 @@ -package session - -import ( - "fmt" - "testing" - "time" - - "github.com/filecoin-project/lassie/pkg/types" - "github.com/google/uuid" - "github.com/ipfs/go-cid" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" - "github.com/ipni/go-libipni/metadata" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSPConcurrency(t *testing.T) { - state := NewSessionState(DefaultConfig()) - selector := selectorparse.CommonSelector_ExploreAllRecursively - ret1 := types.RetrievalID(uuid.New()) - cid1 := cid.MustParse("bafkqaalb") - ret2 := types.RetrievalID(uuid.New()) - cid2 := cid.MustParse("bafkqaalc") - ret3 := types.RetrievalID(uuid.New()) - cid3 := cid.MustParse("bafkqaald") - p1 := peer.ID("A") - p2 := peer.ID("B") - p3 := peer.ID("C") - - assert.True(t, state.RegisterRetrieval(ret1, cid1, selector)) - - require.Equal(t, uint(0), state.GetConcurrency(p1)) - require.Equal(t, uint(0), state.GetConcurrency(p2)) - require.Equal(t, uint(0), state.GetConcurrency(p3)) - - require.NoError(t, state.AddToRetrieval(ret1, []peer.ID{p1, p2, p3})) - require.Error(t, state.AddToRetrieval(ret2, []peer.ID{p1, p2, p3})) // no such retrieval (yet) - - require.Equal(t, uint(1), state.GetConcurrency(p1)) - require.Equal(t, uint(1), state.GetConcurrency(p2)) - require.Equal(t, uint(1), state.GetConcurrency(p3)) - - assert.True(t, state.RegisterRetrieval(ret2, cid2, selector)) - require.NoError(t, state.AddToRetrieval(ret2, []peer.ID{p1, p2})) - - require.Equal(t, uint(2), state.GetConcurrency(p1)) - require.Equal(t, uint(2), state.GetConcurrency(p2)) - require.Equal(t, uint(1), state.GetConcurrency(p3)) - - assert.True(t, state.RegisterRetrieval(ret3, cid3, selector)) - require.NoError(t, state.AddToRetrieval(ret3, []peer.ID{p1})) - - require.Equal(t, uint(3), state.GetConcurrency(p1)) - require.Equal(t, uint(2), state.GetConcurrency(p2)) - require.Equal(t, uint(1), state.GetConcurrency(p3)) - - assert.NoError(t, state.EndRetrieval(ret1)) - require.Equal(t, uint(2), state.GetConcurrency(p1)) - require.Equal(t, uint(1), state.GetConcurrency(p2)) - require.Equal(t, uint(0), state.GetConcurrency(p3)) - - assert.NoError(t, state.EndRetrieval(ret2)) - require.Equal(t, uint(1), state.GetConcurrency(p1)) - require.Equal(t, uint(0), state.GetConcurrency(p2)) - require.Equal(t, uint(0), state.GetConcurrency(p3)) - - assert.NoError(t, state.EndRetrieval(ret3)) - require.Equal(t, uint(0), state.GetConcurrency(p1)) - require.Equal(t, uint(0), state.GetConcurrency(p2)) - require.Equal(t, uint(0), state.GetConcurrency(p3)) - - // test failures reducing concurrency - assert.True(t, state.RegisterRetrieval(ret1, cid1, selector)) - assert.True(t, state.RegisterRetrieval(ret2, cid2, selector)) - require.NoError(t, state.AddToRetrieval(ret1, []peer.ID{p1, p2, p3})) - require.NoError(t, state.AddToRetrieval(ret2, []peer.ID{p1, p2, p3})) - - require.Equal(t, uint(2), state.GetConcurrency(p1)) - require.Equal(t, uint(2), state.GetConcurrency(p2)) - require.Equal(t, uint(2), state.GetConcurrency(p3)) - - require.NoError(t, state.RecordFailure(ret1, p1)) - require.Equal(t, uint(1), state.GetConcurrency(p1)) - require.Equal(t, uint(2), state.GetConcurrency(p2)) - require.Equal(t, uint(2), state.GetConcurrency(p3)) - require.ErrorContains(t, state.RecordFailure(ret1, p1), "no such storage provider") - - assert.NoError(t, state.EndRetrieval(ret2)) - require.Equal(t, uint(0), state.GetConcurrency(p1)) - require.Equal(t, uint(1), state.GetConcurrency(p2)) - require.Equal(t, uint(1), state.GetConcurrency(p3)) - - assert.NoError(t, state.EndRetrieval(ret1)) - require.Equal(t, uint(0), state.GetConcurrency(p1)) - require.Equal(t, uint(0), state.GetConcurrency(p2)) - require.Equal(t, uint(0), state.GetConcurrency(p3)) -} - -func TestRetrievalUniqueness(t *testing.T) { - state := NewSessionState(DefaultConfig()) - ret1 := types.RetrievalID(uuid.New()) - cid1 := cid.MustParse("bafkqaalb") - ret2 := types.RetrievalID(uuid.New()) - cid2 := cid.MustParse("bafkqaalc") - ret3 := types.RetrievalID(uuid.New()) - all := selectorparse.CommonSelector_ExploreAllRecursively - matcher := selectorparse.CommonSelector_MatchPoint - - // unique RetrievalID and unique CID; i.e. can't retrieve the same CID simultaneously - assert.True(t, state.RegisterRetrieval(ret1, cid1, all)) - assert.False(t, state.RegisterRetrieval(ret1, cid1, all)) - assert.False(t, state.RegisterRetrieval(ret1, cid2, all)) - assert.False(t, state.RegisterRetrieval(ret2, cid1, all)) - - // unique Retrieval ID can register for a different selector w/ same cid - assert.True(t, state.RegisterRetrieval(ret3, cid1, matcher)) - - require.NoError(t, state.EndRetrieval(ret1)) - require.Error(t, state.EndRetrieval(ret1)) - require.Error(t, state.EndRetrieval(ret2)) - require.NoError(t, state.EndRetrieval(ret3)) - - assert.True(t, state.RegisterRetrieval(ret2, cid1, all)) - assert.False(t, state.RegisterRetrieval(ret2, cid1, all)) - assert.True(t, state.RegisterRetrieval(ret1, cid2, all)) - assert.False(t, state.RegisterRetrieval(ret2, cid1, all)) - - require.NoError(t, state.EndRetrieval(ret1)) - require.NoError(t, state.EndRetrieval(ret2)) - require.Error(t, state.EndRetrieval(ret1)) - require.Error(t, state.EndRetrieval(ret2)) -} - -var retrievalId = types.RetrievalID(uuid.New()) - -type actionType int - -const ( - connectAction actionType = iota - successAction - failureAction - ttfbAction -) - -type action struct { - p peer.ID - typ actionType - d time.Duration - v uint64 -} - -func (a action) execute(t *testing.T, s *SessionState) { - switch a.typ { - case connectAction: - s.RecordConnectTime(a.p, a.d) - case successAction: - s.RecordSuccess(a.p, a.v) - case failureAction: - require.NoError(t, s.AddToRetrieval(retrievalId, []peer.ID{a.p})) - require.NoError(t, s.RecordFailure(retrievalId, a.p)) - case ttfbAction: - s.RecordFirstByteTime(a.p, a.d) - default: - panic("unrecognized action type") - } -} - -/* -The following tests use an implicit understanding of the way scoring works -within the State. By controlling a sequence of actions and estimating the -internal scoring, we then used a fixed dice-roll to make sure that each two -providers compare in the predicted way. - -To approximate the expected scoring components, use a Node.js repl, creating an -`ema` function: - ema=(a,r,...v)=>v.reduce((p,c)=>(1-a)*c+a*p,r) -Then use it to calculate the per-score, per-provider, and across-provider -components: - - connect time (seconds->milliseconds, truncated to ints), `a` of 0.5: - Math.floor(ema(0.5, 10000, 11000, 13000)) = 1175 - - to calculate overall (oema) of connect time, use an `a` of 0.8, for all - values, e.g.: - Math.floor(ema(0.8, 10000, 11000, 13000, 13000, 12000, 12000, 12000)) - - Note that internally, the ema function in Go will truncate the millisecond - integers per calculation, so the results won't be quite the same in JS. An - int ema calculator would be more accurate would be: - emaI=(a,r,...v)=>v.reduce((p,c)=>Math.floor((1-a)*c+a*p),r) -To calculate the decayed normalised form of connect time: - - Math.exp(-(1/oema)*ema) -*/ - -func TestCandidateComparison(t *testing.T) { - peers := make([]peer.ID, 10) - for i := 0; i < 10; i++ { - peers[i] = peer.ID(fmt.Sprintf("peer%d", i)) - } - - testCases := []struct { - name string - actions []action - metadata map[peer.ID]metadata.Protocol - expectedOrder []peer.ID - }{ - { - name: "http peers, different connect speeds", - actions: []action{ - {p: peers[0], typ: connectAction, d: time.Second}, - {p: peers[1], typ: connectAction, d: 2 * time.Second}, - {p: peers[2], typ: connectAction, d: 3 * time.Second}, - {p: peers[3], typ: connectAction, d: 4 * time.Second}, - {p: peers[4], typ: connectAction, d: 5 * time.Second}, - }, - expectedOrder: []peer.ID{peers[0], peers[1], peers[2], peers[3], peers[4]}, - }, - { - name: "graphsync peers, different connect speeds", - actions: []action{ - {p: peers[0], typ: connectAction, d: time.Second}, - {p: peers[1], typ: connectAction, d: 2 * time.Second}, - {p: peers[2], typ: connectAction, d: 3 * time.Second}, - {p: peers[3], typ: connectAction, d: 4 * time.Second}, - {p: peers[4], typ: connectAction, d: 5 * time.Second}, - }, - metadata: map[peer.ID]metadata.Protocol{ - peers[0]: &metadata.GraphsyncFilecoinV1{}, - peers[1]: &metadata.GraphsyncFilecoinV1{}, - peers[2]: &metadata.GraphsyncFilecoinV1{}, - peers[3]: &metadata.GraphsyncFilecoinV1{}, - peers[4]: &metadata.GraphsyncFilecoinV1{}, - }, - expectedOrder: []peer.ID{peers[0], peers[1], peers[2], peers[3], peers[4]}, - }, - { - name: "graphsync chooses best, w/ connect time", - actions: []action{ - {p: peers[0], typ: connectAction, d: time.Second}, - {p: peers[1], typ: connectAction, d: time.Second}, - {p: peers[2], typ: connectAction, d: 2 * time.Second}, - {p: peers[3], typ: connectAction, d: 2 * time.Second}, - {p: peers[4], typ: connectAction, d: 3 * time.Second}, - }, - metadata: map[peer.ID]metadata.Protocol{ - peers[0]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: true, FastRetrieval: false}, - peers[1]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[2]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[3]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, - peers[4]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, // same as prev, slower connect - }, - expectedOrder: []peer.ID{peers[0], peers[1], peers[2], peers[3], peers[4]}, - }, - { - name: "multiple connect, averages don't cross", - actions: []action{ - {p: peers[0], typ: connectAction, d: 10 * time.Second}, // c-ema: 10000 - {p: peers[0], typ: connectAction, d: 11 * time.Second}, // c-ema: 10500 - {p: peers[0], typ: connectAction, d: 13 * time.Second}, // c-ema: 11750 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - // connect oema: 11364 - }, - expectedOrder: []peer.ID{peers[0], peers[1]}, - }, - { - name: "multiple connect, averages cross", - actions: []action{ - {p: peers[0], typ: connectAction, d: 10 * time.Second}, // c-ema: 10000 - {p: peers[0], typ: connectAction, d: 11 * time.Second}, // c-ema: 10500 - {p: peers[0], typ: connectAction, d: 13 * time.Second}, // c-ema: 11750 - {p: peers[0], typ: connectAction, d: 13 * time.Second}, // c-ema: 12375 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - // connect oema: 11593 - }, - expectedOrder: []peer.ID{peers[1], peers[0]}, - }, - { - // a peer without connect time data should use the oema - name: "no connect data, first", - actions: []action{ - {p: peers[0], typ: connectAction, d: 10 * time.Second}, // c-ema: 10000 - {p: peers[0], typ: connectAction, d: 11 * time.Second}, // c-ema: 10500 - {p: peers[0], typ: connectAction, d: 13 * time.Second}, // c-ema: 11750 - {p: peers[0], typ: connectAction, d: 13 * time.Second}, // c-ema: 12375 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - // connect oema: 11593, peer[2] will use this value - }, - expectedOrder: []peer.ID{peers[2], peers[1], peers[0]}, - }, - { - // same as previous but we're expecting it to slot in later - name: "no connect data, not first", - actions: []action{ - {p: peers[0], typ: connectAction, d: 10 * time.Second}, // c-ema: 10000 - {p: peers[0], typ: connectAction, d: 11 * time.Second}, // c-ema: 10500 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - {p: peers[1], typ: connectAction, d: 12 * time.Second}, // c-ema: 12000 - // connect oema: 11078, peer[2] will use this value - }, - expectedOrder: []peer.ID{peers[0], peers[2], peers[1]}, - }, - { - name: "success better than failure", - actions: []action{ - {p: peers[1], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[0], typ: failureAction, d: 0}, // s-ema: 0 - }, - expectedOrder: []peer.ID{peers[1], peers[0]}, - }, - { - name: "success, flakies, failure", - actions: []action{ - {p: peers[1], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[1], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[3], typ: failureAction, d: 0}, // s-ema: 0 - {p: peers[3], typ: failureAction, d: 0}, // s-ema: 0 - {p: peers[0], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[0], typ: failureAction, d: 0}, // s-ema: 0.5 - {p: peers[0], typ: successAction, v: 0}, // s-ema: 0.75 - {p: peers[0], typ: failureAction, d: 0}, // s-ema: 0.375 - {p: peers[2], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[2], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[2], typ: failureAction, d: 0}, // s-ema: 0.5 - }, - expectedOrder: []peer.ID{peers[1], peers[2], peers[0], peers[3]}, - }, - { - name: "combined metrics", - actions: []action{ - // 0, 1, 2 - {p: peers[0], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[0], typ: connectAction, d: 1 * time.Second}, // s-ema: 1, c-ema: 10000 - {p: peers[1], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[1], typ: connectAction, d: 2 * time.Second}, // s-ema: 1, c-ema: 20000 - {p: peers[2], typ: failureAction, d: 0}, // s-ema: 0 - {p: peers[2], typ: connectAction, d: 1 * time.Second}, // s-ema: 0, c-ema: 10000 - - // same pattern, better metadata: 5, 4, 3 - {p: peers[5], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[5], typ: connectAction, d: 1 * time.Second}, // s-ema: 1, c-ema: 10000 - {p: peers[4], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[4], typ: connectAction, d: 2 * time.Second}, // s-ema: 1, c-ema: 20000 - {p: peers[3], typ: failureAction, d: 0}, // s-ema: 0 - {p: peers[3], typ: connectAction, d: 1 * time.Second}, // s-ema: 0, c-ema: 10000 - - // same pattern, best metadata: 7, 8, 6 - {p: peers[7], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[7], typ: connectAction, d: 1 * time.Second}, // s-ema: 1, c-ema: 10000 - {p: peers[8], typ: successAction, v: 0}, // s-ema: 1 - {p: peers[8], typ: connectAction, d: 2 * time.Second}, // s-ema: 1, c-ema: 20000 - {p: peers[6], typ: failureAction, d: 0}, // s-ema: 0 - {p: peers[6], typ: connectAction, d: 1 * time.Second}, // s-ema: 0, c-ema: 10000 - - // connect oema: 1282 - }, - metadata: map[peer.ID]metadata.Protocol{ - peers[0]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, - peers[1]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, - peers[2]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, - - peers[3]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[4]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[5]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - - peers[6]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: true, FastRetrieval: true}, - peers[7]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: true, FastRetrieval: true}, - peers[8]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: true, FastRetrieval: true}, - }, - expectedOrder: []peer.ID{peers[7], peers[8], peers[6], peers[5], peers[4], peers[3], peers[0], peers[1], peers[2]}, - }, - { - name: "ttfb chooses best", - actions: []action{ - {p: peers[4], typ: ttfbAction, d: time.Second}, - {p: peers[3], typ: ttfbAction, d: 2 * time.Second}, - {p: peers[2], typ: ttfbAction, d: 3 * time.Second}, - {p: peers[1], typ: ttfbAction, d: 4 * time.Second}, - {p: peers[0], typ: ttfbAction, d: 5 * time.Second}, - }, - expectedOrder: []peer.ID{peers[4], peers[3], peers[2], peers[1], peers[0]}, - }, - { - name: "graphsync chooses best, w/ ttfb time", - actions: []action{ - {p: peers[0], typ: ttfbAction, d: time.Second}, - {p: peers[1], typ: ttfbAction, d: time.Second}, - {p: peers[2], typ: ttfbAction, d: 2 * time.Second}, - {p: peers[3], typ: ttfbAction, d: 2 * time.Second}, - {p: peers[4], typ: ttfbAction, d: 3 * time.Second}, - }, - metadata: map[peer.ID]metadata.Protocol{ - peers[0]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: true, FastRetrieval: false}, - peers[1]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[2]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[3]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, - peers[4]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, // same as prev, slower connect - }, - expectedOrder: []peer.ID{peers[0], peers[1], peers[2], peers[3], peers[4]}, - }, - { - name: "multiple ttfb, averages don't cross", - actions: []action{ - {p: peers[0], typ: ttfbAction, d: 10 * time.Second}, // ttfb-ema: 10000 - {p: peers[0], typ: ttfbAction, d: 11 * time.Second}, // ttfb-ema: 10500 - {p: peers[0], typ: ttfbAction, d: 13 * time.Second}, // ttfb-ema: 11750 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - // ttfb oema: 11364 - }, - expectedOrder: []peer.ID{peers[0], peers[1]}, - }, - { - name: "multiple ttfb, averages cross", - actions: []action{ - {p: peers[0], typ: ttfbAction, d: 10 * time.Second}, // ttfb-ema: 10000 - {p: peers[0], typ: ttfbAction, d: 11 * time.Second}, // ttfb-ema: 10500 - {p: peers[0], typ: ttfbAction, d: 13 * time.Second}, // ttfb-ema: 11750 - {p: peers[0], typ: ttfbAction, d: 13 * time.Second}, // ttfb-ema: 12375 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - // ttfb oema: 11593 - }, - expectedOrder: []peer.ID{peers[1], peers[0]}, - }, - { - // a peer without ttfb time data should use the oema - name: "no ttfb data, first", - actions: []action{ - {p: peers[0], typ: ttfbAction, d: 10 * time.Second}, // ttfb-ema: 10000 - {p: peers[0], typ: ttfbAction, d: 11 * time.Second}, // ttfb-ema: 10500 - {p: peers[0], typ: ttfbAction, d: 13 * time.Second}, // ttfb-ema: 11750 - {p: peers[0], typ: ttfbAction, d: 13 * time.Second}, // ttfb-ema: 12375 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - // ttfb oema: 11593, peer[2] will use this value - }, - expectedOrder: []peer.ID{peers[2], peers[1], peers[0]}, - }, - { - // same as previous but we're expecting it to slot in later - name: "no ttfb data, not first", - actions: []action{ - {p: peers[0], typ: ttfbAction, d: 10 * time.Second}, // ttfb-ema: 10000 - {p: peers[0], typ: ttfbAction, d: 11 * time.Second}, // ttfb-ema: 10500 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - {p: peers[1], typ: ttfbAction, d: 12 * time.Second}, // ttfb-ema: 12000 - // ttfb oema: 11078, peer[2] will use this value - }, - expectedOrder: []peer.ID{peers[0], peers[2], peers[1]}, - }, - { - name: "bandwidth chooses best", - actions: []action{ - {p: peers[4], typ: successAction, v: 1}, - {p: peers[3], typ: successAction, v: 2}, - {p: peers[2], typ: successAction, v: 3}, - {p: peers[1], typ: successAction, v: 4}, - {p: peers[0], typ: successAction, v: 5}, - }, - expectedOrder: []peer.ID{peers[4], peers[3], peers[2], peers[1], peers[0]}, - }, - { - name: "graphsync chooses best, w/ bandwidth time", - actions: []action{ - {p: peers[0], typ: successAction, v: 1}, - {p: peers[1], typ: successAction, v: 1}, - {p: peers[2], typ: successAction, v: 2}, - {p: peers[3], typ: successAction, v: 2}, - {p: peers[4], typ: successAction, v: 3}, - }, - metadata: map[peer.ID]metadata.Protocol{ - peers[0]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: true, FastRetrieval: false}, - peers[1]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[2]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: true}, - peers[3]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, - peers[4]: &metadata.GraphsyncFilecoinV1{VerifiedDeal: false, FastRetrieval: false}, // same as prev, slower connect - }, - expectedOrder: []peer.ID{peers[0], peers[1], peers[2], peers[3], peers[4]}, - }, - { - name: "multiple bandwidth, averages don't cross", - actions: []action{ - {p: peers[0], typ: successAction, v: 1000}, // b-ema: 10000 - {p: peers[0], typ: successAction, v: 1100}, // b-ema: 10500 - {p: peers[0], typ: successAction, v: 1300}, // b-ema: 11750 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - // bandwidth oema: 11364 - }, - expectedOrder: []peer.ID{peers[0], peers[1]}, - }, - { - name: "multiple bandwidth, averages cross", - actions: []action{ - {p: peers[0], typ: successAction, v: 1000}, // b-ema: 10000 - {p: peers[0], typ: successAction, v: 1100}, // b-ema: 10500 - {p: peers[0], typ: successAction, v: 1300}, // b-ema: 11750 - {p: peers[0], typ: successAction, v: 1300}, // b-ema: 12375 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - // bandwidth oema: 11593 - }, - expectedOrder: []peer.ID{peers[1], peers[0]}, - }, - { - // a peer without bandwidth time data should use the oema - name: "no bandwidth data, first", - actions: []action{ - {p: peers[0], typ: successAction, v: 1000}, // b-ema: 10000 - {p: peers[0], typ: successAction, v: 1100}, // b-ema: 10500 - {p: peers[0], typ: successAction, v: 1300}, // b-ema: 11750 - {p: peers[0], typ: successAction, v: 1300}, // b-ema: 12375 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - // bandwidth oema: 11593, peer[2] will use this value - }, - expectedOrder: []peer.ID{peers[2], peers[1], peers[0]}, - }, - { - // same as previous but we're expecting it to slot in later - name: "no bandwidth data, not first", - actions: []action{ - {p: peers[0], typ: successAction, v: 1000}, // b-ema: 10000 - {p: peers[0], typ: successAction, v: 1100}, // b-ema: 10500 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - {p: peers[1], typ: successAction, v: 1200}, // b-ema: 12000 - // bandwidth oema: 11078, peer[2] will use this value - }, - expectedOrder: []peer.ID{peers[0], peers[2], peers[1]}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - cfg := DefaultConfig().WithoutRandomness() - state := NewSessionState(cfg) - // setup a retrieval so we don't error on "unknown retrieval" - require.True(t, state.RegisterRetrieval(retrievalId, cid.Undef, basicnode.NewString("boop"))) - - for _, action := range tc.actions { - action.execute(t, state) - } - - // list of peers we're working with, build it in the same order as - // the peers array so we don't bias with 'expectedOrder' ordering - tp := make([]peer.ID, 0, len(tc.expectedOrder)) - mda := make([]metadata.Protocol, 0, len(tc.expectedOrder)) - for len(tp) < len(tc.expectedOrder) { - for _, p := range tc.expectedOrder { - if p == peers[len(tp)] { - if len(tc.metadata) > 0 { - mda = append(mda, tc.metadata[p]) - } else { - mda = append(mda, metadata.IpfsGatewayHttp{}) - } - tp = append(tp, p) - break - } - } - } - - // choose next provider until we've exhausted the list - gotOrder := make([]peer.ID, 0, len(tc.expectedOrder)) - for len(gotOrder) < len(tc.expectedOrder) { - next := state.ChooseNextProvider(tp, mda) - gotOrder = append(gotOrder, tp[next]) - tp = append(tp[:next], tp[next+1:]...) - } - - require.Equal(t, tc.expectedOrder, gotOrder) - }) - } -} diff --git a/pkg/storage/cachingtempstore.go b/pkg/storage/cachingtempstore.go index c5d03142..dc07a392 100644 --- a/pkg/storage/cachingtempstore.go +++ b/pkg/storage/cachingtempstore.go @@ -9,13 +9,11 @@ import ( "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" - carstorage "github.com/ipld/go-car/v2/storage" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) var _ types.ReadableWritableStorage = (*CachingTempStore)(nil) -var _ types.ReadableWritableStorage = (*preloadStore)(nil) var errClosed = errors.New("store closed") @@ -26,25 +24,15 @@ var errClosed = errors.New("store closed") // // A provided BlockWriteOpener will receive blocks for each Put operation, this // is intended to be used to write a properly ordered CARv1 file. -// -// PreloadStore returns a secondary ReadableWritableStorage that can be used -// by a traversal preloader to optimistically load blocks into the temporary -// store. Blocks loaded via the PreloadStore will not be written to the -// provided BlockWriteOpener until they appear in a Put operation on the parent -// store. In this way, the BlockWriteOpener will receive blocks in the order -// that they appear in the traversal. type CachingTempStore struct { store *DeferredStorageCar outWriter linking.BlockWriteOpener - - preloadKeys map[string]struct{} } func NewCachingTempStore(outWriter linking.BlockWriteOpener, store *DeferredStorageCar) *CachingTempStore { return &CachingTempStore{ - store: store, - outWriter: outWriter, - preloadKeys: make(map[string]struct{}), + store: store, + outWriter: outWriter, } } @@ -52,11 +40,6 @@ func (ttrw *CachingTempStore) Has(ctx context.Context, key string) (bool, error) ttrw.store.lk.Lock() defer ttrw.store.lk.Unlock() - if _, ok := ttrw.preloadKeys[key]; ok { - // if it's in the preload list, then it's not in the store proper - return false, nil - } - if rw, err := ttrw.store.readWrite(); err != nil { return false, err } else { @@ -68,15 +51,6 @@ func (ttrw *CachingTempStore) Get(ctx context.Context, key string) ([]byte, erro ttrw.store.lk.Lock() defer ttrw.store.lk.Unlock() - if _, ok := ttrw.preloadKeys[key]; ok { - // if it's in the preload list, then it's not in the store proper - c, err := cid.Cast([]byte(key)) - if err != nil { - return nil, err - } - return nil, carstorage.ErrNotFound{Cid: c} - } - if rw, err := ttrw.store.readWrite(); err != nil { return nil, err } else { @@ -88,20 +62,10 @@ func (ttrw *CachingTempStore) GetStream(ctx context.Context, key string) (io.Rea ttrw.store.lk.Lock() defer ttrw.store.lk.Unlock() - if _, ok := ttrw.preloadKeys[key]; ok { - // if it's in the preload list, then it's not in the store proper - c, err := cid.Cast([]byte(key)) - if err != nil { - return nil, err - } - return nil, carstorage.ErrNotFound{Cid: c} - } - if rw, err := ttrw.store.readWrite(); err != nil { return nil, err } else { - r, err := rw.GetStream(ctx, key) - return r, err + return rw.GetStream(ctx, key) } } @@ -110,13 +74,6 @@ func (ttrw *CachingTempStore) GetStream(ctx context.Context, key string) (io.Rea func (ttrw *CachingTempStore) Put(ctx context.Context, key string, data []byte) error { ttrw.store.lk.Lock() defer ttrw.store.lk.Unlock() - - if _, ok := ttrw.preloadKeys[key]; ok { - // already in preload, just write to the outWriter - delete(ttrw.preloadKeys, key) - return writeTo(ctx, ttrw.outWriter, key, data) - } - // not in preload, write to local and outWriter return ttrw.teePut(ctx, key, data) } @@ -175,76 +132,3 @@ func writeTo(ctx context.Context, outWriter linking.BlockWriteOpener, key string } return c(cidlink.Link{Cid: cid}) } - -func (ttrw *CachingTempStore) PreloadStore() types.ReadableWritableStorage { - return &preloadStore{ttrw: ttrw} -} - -type preloadStore struct { - ttrw *CachingTempStore -} - -func (ps *preloadStore) Has(ctx context.Context, key string) (bool, error) { - ps.ttrw.store.lk.Lock() - defer ps.ttrw.store.lk.Unlock() - _, has := ps.ttrw.preloadKeys[key] - return has, nil -} - -func (ps *preloadStore) Get(ctx context.Context, key string) ([]byte, error) { - ps.ttrw.store.lk.Lock() - defer ps.ttrw.store.lk.Unlock() - if _, ok := ps.ttrw.preloadKeys[key]; !ok { - c, err := cid.Cast([]byte(key)) - if err != nil { - return nil, err - } - return nil, carstorage.ErrNotFound{Cid: c} - } - if rw, err := ps.ttrw.store.readWrite(); err != nil { - return nil, err - } else { - return rw.Get(ctx, key) - } -} - -func (ps *preloadStore) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { - ps.ttrw.store.lk.Lock() - defer ps.ttrw.store.lk.Unlock() - if _, ok := ps.ttrw.preloadKeys[key]; !ok { - c, err := cid.Cast([]byte(key)) - if err != nil { - return nil, err - } - return nil, carstorage.ErrNotFound{Cid: c} - } - if rw, err := ps.ttrw.store.readWrite(); err != nil { - return nil, err - } else { - return rw.GetStream(ctx, key) - } -} - -func (ps *preloadStore) Put(ctx context.Context, key string, data []byte) error { - ps.ttrw.store.lk.Lock() - defer ps.ttrw.store.lk.Unlock() - // is it already in the preload list? - if _, ok := ps.ttrw.preloadKeys[key]; ok { - return nil - } - if rw, err := ps.ttrw.store.readWrite(); err != nil { - return err - } else { - // do we already have it in the store? - if has, err := rw.Has(ctx, key); err != nil { - return err - } else if has { - return nil - } - if err := rw.Put(ctx, key, data); err != nil { - return err - } - ps.ttrw.preloadKeys[key] = struct{}{} - return nil - } -} diff --git a/pkg/storage/cachingtempstore_test.go b/pkg/storage/cachingtempstore_test.go index 993c4221..251ef1d7 100644 --- a/pkg/storage/cachingtempstore_test.go +++ b/pkg/storage/cachingtempstore_test.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "io" - "math/rand" + "math/rand/v2" "sync" "testing" "time" @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" ) -var rng = rand.New(rand.NewSource(3333)) +var rng = rand.NewChaCha8([32]byte{33, 33, 33, 33}) var rngLk sync.Mutex func TestDeferredCarWriterWritesCARv1(t *testing.T) { diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 6a1d8fd2..c0103623 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -124,95 +124,3 @@ func TestTempCarStorage(t *testing.T) { }) } } - -type blk struct { - cid cid.Cid - data []byte -} - -func TestPreloadStore(t *testing.T) { - ctx := context.Background() - - td := make([]blk, 0, 10) - for i := 0; i < 10; i++ { - c, d := randBlock() - td = append(td, blk{c, d}) - } - - bwoCnt := 0 - bwo := func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { - var buf bytes.Buffer - return &buf, func(lnk ipld.Link) error { - c := lnk.(cidlink.Link).Cid - if bwoCnt >= len(td) { - require.Fail(t, "too many calls to bwo") - } else { - require.Equal(t, td[bwoCnt].cid, c) - require.Equal(t, td[bwoCnt].data, buf.Bytes()) - } - bwoCnt++ - return nil - }, nil - } - mainStore := NewCachingTempStore(bwo, NewDeferredStorageCar(t.TempDir(), td[0].cid)) - t.Cleanup(func() { - require.NoError(t, mainStore.Close()) - }) - preload := mainStore.PreloadStore() - - checkNotHas := func(blks []blk, stores ...types.ReadableWritableStorage) { - for _, d := range blks { - for _, s := range stores { - has, err := s.Has(ctx, d.cid.KeyString()) - require.NoError(t, err) - require.False(t, has) - _, err = s.Get(ctx, d.cid.KeyString()) - require.Error(t, err) - enf, ok := err.(interface{ NotFound() bool }) - require.True(t, ok) - require.True(t, enf.NotFound()) - _, err = s.GetStream(ctx, d.cid.KeyString()) - require.Error(t, err) - enf, ok = err.(interface{ NotFound() bool }) - require.True(t, ok) - require.True(t, enf.NotFound()) - } - } - } - - checkHas := func(blks []blk, stores ...types.ReadableWritableStorage) { - for _, d := range blks { - for _, s := range stores { - has, err := s.Has(ctx, d.cid.KeyString()) - require.NoError(t, err) - require.True(t, has) - got, err := s.Get(ctx, d.cid.KeyString()) - require.NoError(t, err) - require.Equal(t, d.data, got) - gotStream, err := s.GetStream(ctx, d.cid.KeyString()) - require.NoError(t, err) - got, err = io.ReadAll(gotStream) - require.NoError(t, err) - require.Equal(t, d.data, got) - } - } - } - - checkNotHas(td, mainStore, preload) - mainStore.Put(ctx, td[0].cid.KeyString(), td[0].data) - checkNotHas(td, preload) - checkNotHas(td[1:], mainStore) - checkHas(td[:1], mainStore) - for i := 5; i >= 1; i-- { // out of order, partial preload - preload.Put(ctx, td[i].cid.KeyString(), td[i].data) - } - checkNotHas(td[1:], mainStore) - checkNotHas(td[:1], preload) - checkHas(td[1:6], preload) - checkNotHas(td[6:], preload) - for _, d := range td[1:] { // in order, complete - mainStore.Put(ctx, d.cid.KeyString(), d.data) - } - checkHas(td, mainStore) - checkNotHas(td, preload) -} diff --git a/pkg/storage/streamingstore.go b/pkg/storage/streamingstore.go new file mode 100644 index 00000000..2d0defb7 --- /dev/null +++ b/pkg/storage/streamingstore.go @@ -0,0 +1,137 @@ +package storage + +import ( + "bytes" + "context" + "io" + "sync" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + carstorage "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +var _ types.ReadableWritableStorage = (*StreamingStore)(nil) + +// StreamingStore writes blocks directly to output without retaining data. +// Get() always returns NotFound; Has() checks the seen set; Put() writes +// to output and marks the CID as seen. +type StreamingStore struct { + seen map[cid.Cid]struct{} + outWriter linking.BlockWriteOpener + mu sync.RWMutex + closed bool +} + +func NewStreamingStore(outWriter linking.BlockWriteOpener) *StreamingStore { + return &StreamingStore{ + seen: make(map[cid.Cid]struct{}), + outWriter: outWriter, + } +} + +func (s *StreamingStore) Has(ctx context.Context, key string) (bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return false, errClosed + } + + c, err := cid.Cast([]byte(key)) + if err != nil { + return false, err + } + + _, ok := s.seen[c] + return ok, nil +} + +func (s *StreamingStore) Get(ctx context.Context, key string) ([]byte, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return nil, errClosed + } + + c, err := cid.Cast([]byte(key)) + if err != nil { + return nil, err + } + + return nil, carstorage.ErrNotFound{Cid: c} +} + +func (s *StreamingStore) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return nil, errClosed + } + + c, err := cid.Cast([]byte(key)) + if err != nil { + return nil, err + } + + return nil, carstorage.ErrNotFound{Cid: c} +} + +func (s *StreamingStore) Put(ctx context.Context, key string, data []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return errClosed + } + + c, err := cid.Cast([]byte(key)) + if err != nil { + return err + } + + if _, ok := s.seen[c]; ok { + return nil + } + + if err := s.writeToOutput(ctx, c, data); err != nil { + return err + } + + s.seen[c] = struct{}{} + return nil +} + +func (s *StreamingStore) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + return nil +} + +func (s *StreamingStore) Seen() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.seen) +} + +func (s *StreamingStore) writeToOutput(ctx context.Context, c cid.Cid, data []byte) error { + w, commit, err := s.outWriter(linking.LinkContext{Ctx: ctx}) + if err != nil { + return err + } + + n, err := bytes.NewBuffer(data).WriteTo(w) + if err != nil { + return err + } + if n != int64(len(data)) { + return io.ErrShortWrite + } + + return commit(cidlink.Link{Cid: c}) +} diff --git a/pkg/storage/streamingstore_test.go b/pkg/storage/streamingstore_test.go new file mode 100644 index 00000000..0e9408fc --- /dev/null +++ b/pkg/storage/streamingstore_test.go @@ -0,0 +1,106 @@ +package storage + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/stretchr/testify/require" +) + +func TestStreamingStore(t *testing.T) { + ctx := context.Background() + testCid1, testData1 := randBlock() + testCid2, testData2 := randBlock() + testCid3, _ := randBlock() + + written := make(map[cid.Cid][]byte) + bwo := func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { + var buf bytes.Buffer + return &buf, func(lnk ipld.Link) error { + written[lnk.(cidlink.Link).Cid] = buf.Bytes() + return nil + }, nil + } + + ss := NewStreamingStore(bwo) + defer ss.Close() + + has, err := ss.Has(ctx, testCid1.KeyString()) + require.NoError(t, err) + require.False(t, has) + require.Equal(t, 0, ss.Seen()) + + // Get/GetStream always return NotFound — streaming doesn't retain data + _, err = ss.Get(ctx, testCid1.KeyString()) + require.Error(t, err) + nf, ok := err.(interface{ NotFound() bool }) + require.True(t, ok) + require.True(t, nf.NotFound()) + + _, err = ss.GetStream(ctx, testCid1.KeyString()) + require.Error(t, err) + nf, ok = err.(interface{ NotFound() bool }) + require.True(t, ok) + require.True(t, nf.NotFound()) + + require.NoError(t, ss.Put(ctx, testCid1.KeyString(), testData1)) + require.Len(t, written, 1) + require.Equal(t, testData1, written[testCid1]) + require.Equal(t, 1, ss.Seen()) + + has, err = ss.Has(ctx, testCid1.KeyString()) + require.NoError(t, err) + require.True(t, has) + + // Get still returns NotFound after Put — data not retained + _, err = ss.Get(ctx, testCid1.KeyString()) + require.Error(t, err) + nf, ok = err.(interface{ NotFound() bool }) + require.True(t, ok) + require.True(t, nf.NotFound()) + + require.NoError(t, ss.Put(ctx, testCid2.KeyString(), testData2)) + require.Len(t, written, 2) + require.Equal(t, testData2, written[testCid2]) + require.Equal(t, 2, ss.Seen()) + + // duplicate Put is a no-op + require.NoError(t, ss.Put(ctx, testCid1.KeyString(), testData1)) + require.Len(t, written, 2) // Still 2 + require.Equal(t, 2, ss.Seen()) + + has, err = ss.Has(ctx, testCid3.KeyString()) + require.NoError(t, err) + require.False(t, has) +} + +func TestStreamingStoreClose(t *testing.T) { + ctx := context.Background() + testCid1, testData1 := randBlock() + + bwo := func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { + var buf bytes.Buffer + return &buf, func(lnk ipld.Link) error { return nil }, nil + } + + ss := NewStreamingStore(bwo) + require.NoError(t, ss.Close()) + + // All operations return errClosed after close + _, err := ss.Has(ctx, testCid1.KeyString()) + require.Equal(t, errClosed, err) + + _, err = ss.Get(ctx, testCid1.KeyString()) + require.Equal(t, errClosed, err) + + _, err = ss.GetStream(ctx, testCid1.KeyString()) + require.Equal(t, errClosed, err) + + err = ss.Put(ctx, testCid1.KeyString(), testData1) + require.Equal(t, errClosed, err) +} diff --git a/pkg/types/request.go b/pkg/types/request.go index faf68042..f98a823d 100644 --- a/pkg/types/request.go +++ b/pkg/types/request.go @@ -1,6 +1,7 @@ package types import ( + "context" "crypto/rand" "errors" "fmt" @@ -28,6 +29,12 @@ type ReadableWritableStorage interface { ipldstorage.StreamingReadableStorage } +// HasChecker is an optional interface for checking if a block exists in storage +// without reading it. Used to avoid redundant fetches in per-block mode. +type HasChecker interface { + Has(ctx context.Context, key string) (bool, error) +} + type RetrievalID uuid.UUID func NewRetrievalID() (RetrievalID, error) { @@ -72,11 +79,6 @@ type RetrievalRequest struct { // If nil, the default protocols will be used. Protocols []multicodec.Code - // PreloadLinkSystem must be setup to enable Bitswap preload behavior. This - // LinkSystem must be thread-safe as multiple goroutines may be using it to - // store and retrieve blocks concurrently. - PreloadLinkSystem ipld.LinkSystem - // MaxBlocks optionally specifies the maximum number of blocks to fetch. // If zero, no limit is applied. MaxBlocks uint64 @@ -213,20 +215,12 @@ func (r RetrievalRequest) GetSupportedProtocols(allSupportedProtocols []multicod return supportedProtocols } -func (r RetrievalRequest) HasPreloadLinkSystem() bool { - return r.PreloadLinkSystem.StorageReadOpener != nil && r.PreloadLinkSystem.StorageWriteOpener != nil -} - func ParseProtocolsString(v string) ([]multicodec.Code, error) { vs := strings.Split(v, ",") protocols := make([]multicodec.Code, 0, len(vs)) for _, v := range vs { var protocol multicodec.Code switch v { - case "bitswap": - protocol = multicodec.TransportBitswap - case "graphsync": - protocol = multicodec.TransportGraphsyncFilecoinv1 case "http": protocol = multicodec.TransportIpfsGatewayHttp default: @@ -265,17 +259,11 @@ func ParseProviderStrings(v string) ([]Provider, error) { var foundProtocol bool for _, part := range parts[1:] { switch part { - case "bitswap": - foundProtocol = true - protocols = append(protocols, metadata.Bitswap{}) - case "graphsync": - foundProtocol = true - protocols = append(protocols, &metadata.GraphsyncFilecoinV1{}) case "http": foundProtocol = true protocols = append(protocols, metadata.IpfsGatewayHttp{}) default: - // if we haven't encountered a prootocol string yet, assume this + was in the multiaddr for some reason + // if we haven't encountered a protocol string yet, assume this + was in the multiaddr for some reason // if we have, something is malconstructed if foundProtocol { return nil, fmt.Errorf("unrecognized protocol: %s", v) @@ -330,6 +318,9 @@ func nextUnknownPeerID() peer.ID { } func IsUnknownPeerID(p peer.ID) bool { + if len(p) < len(unknownPeerID) { + return false + } return p[0:len(unknownPeerID)] == unknownPeerID } @@ -346,12 +337,8 @@ func ToProviderString(ai []Provider) (string, error) { sb.WriteString(ma[0].String()) for _, protocol := range v.Protocols { switch protocol.(type) { - case metadata.Bitswap, *metadata.Bitswap: - sb.WriteString("+bitswap") case metadata.IpfsGatewayHttp, *metadata.IpfsGatewayHttp: sb.WriteString("+http") - case *metadata.GraphsyncFilecoinV1: - sb.WriteString("+graphsync") } } } diff --git a/pkg/types/request_test.go b/pkg/types/request_test.go deleted file mode 100644 index 8794f20e..00000000 --- a/pkg/types/request_test.go +++ /dev/null @@ -1,295 +0,0 @@ -package types - -import ( - "testing" - - "github.com/ipfs/go-cid" - trustlessutils "github.com/ipld/go-trustless-utils" - "github.com/ipni/go-libipni/metadata" - "github.com/multiformats/go-multicodec" - "github.com/stretchr/testify/require" -) - -var testCidV1 = cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") -var testCidV0 = cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK") - -func TestRequestStringRepresentations(t *testing.T) { - // some of the parts of this test are duplicated in go-trustless-utils/tyeps_test.go - - testCases := []struct { - name string - request RetrievalRequest - expectedUrlPath string - expectedDescriptor string - }{ - { - name: "plain", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - }, - expectedUrlPath: "?dag-scope=all", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n", - }, - { - name: "path", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1, Path: "/some/path/to/thing"}, - }, - expectedUrlPath: "/some/path/to/thing?dag-scope=all", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/some/path/to/thing?dag-scope=all&dups=n", - }, - { - name: "escaped path", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1, Path: "/?/#/;/&/ /!"}, - }, - expectedUrlPath: "/%3F/%23/%3B/&/%20/%21?dag-scope=all", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/%3F/%23/%3B/&/%20/%21?dag-scope=all&dups=n", - }, - { - name: "entity", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1, Scope: trustlessutils.DagScopeEntity}, - }, - expectedUrlPath: "?dag-scope=entity", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=entity&dups=n", - }, - { - name: "block", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1, Scope: trustlessutils.DagScopeBlock}, - }, - expectedUrlPath: "?dag-scope=block", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=block&dups=n", - }, - { - name: "protocol", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV0}, - Protocols: []multicodec.Code{multicodec.TransportGraphsyncFilecoinv1}, - }, - expectedUrlPath: "?dag-scope=all", - expectedDescriptor: "/ipfs/QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK?dag-scope=all&dups=n&protocols=transport-graphsync-filecoinv1", - }, - { - name: "protocols", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - Protocols: []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportIpfsGatewayHttp}, - }, - expectedUrlPath: "?dag-scope=all", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n&protocols=transport-bitswap,transport-ipfs-gateway-http", - }, - { - name: "duplicates", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV0, Duplicates: true}, - }, - expectedUrlPath: "?dag-scope=all", - expectedDescriptor: "/ipfs/QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK?dag-scope=all&dups=y", - }, - { - name: "block limit", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - MaxBlocks: 100, - }, - expectedUrlPath: "?dag-scope=all", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n&blockLimit=100", - }, - { - name: "fixed peer", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - Providers: must(ParseProviderStrings("/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4")), - }, - expectedUrlPath: "?dag-scope=all", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n&providers=/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", - }, - { - name: "fixed peers", - request: RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - Providers: must(ParseProviderStrings("/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg,/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4")), - }, - expectedUrlPath: "?dag-scope=all", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n&providers=/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg,/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", - }, - { - name: "byte range", - request: RetrievalRequest{ - Request: trustlessutils.Request{ - Root: testCidV1, - Bytes: &trustlessutils.ByteRange{From: 100, To: ptr(200)}, - }, - }, - expectedUrlPath: "?dag-scope=all&entity-bytes=100:200", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&entity-bytes=100:200&dups=n", - }, - { - name: "byte range -ve", - request: RetrievalRequest{ - Request: trustlessutils.Request{ - Root: testCidV1, - Bytes: &trustlessutils.ByteRange{From: -100}, - }, - }, - expectedUrlPath: "?dag-scope=all&entity-bytes=-100:*", - expectedDescriptor: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&entity-bytes=-100:*&dups=n", - }, - { - name: "all the things", - request: RetrievalRequest{ - Request: trustlessutils.Request{ - Root: testCidV0, - Path: "/some/path/to/thing", - Scope: trustlessutils.DagScopeEntity, - Duplicates: true, - Bytes: &trustlessutils.ByteRange{From: 100, To: ptr(-200)}, - }, - MaxBlocks: 222, - Protocols: []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportIpfsGatewayHttp}, - Providers: must(ParseProviderStrings("/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg,/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4")), - }, - expectedUrlPath: "/some/path/to/thing?dag-scope=entity&entity-bytes=100:-200", - expectedDescriptor: "/ipfs/QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK/some/path/to/thing?dag-scope=entity&entity-bytes=100:-200&dups=y&blockLimit=222&protocols=transport-bitswap,transport-ipfs-gateway-http&providers=/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg,/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual, err := tc.request.Request.UrlPath() - require.NoError(t, err) - require.Equal(t, tc.expectedUrlPath, actual) - actual, err = tc.request.GetDescriptorString() - require.NoError(t, err) - require.Equal(t, tc.expectedDescriptor, actual) - }) - } - - t.Run("fixed peer, no peer ID", func(t *testing.T) { - pps, err := ParseProviderStrings("/ip4/127.0.0.1/tcp/5000/http") - require.NoError(t, err) - request := RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - Providers: pps, - } - ds, err := request.GetDescriptorString() - require.NoError(t, err) - expectedStart := "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n&providers=/ip4/127.0.0.1/tcp/5000/http/p2p/1TunknownX" - require.Equal(t, expectedStart, ds[0:len(expectedStart)]) - }) - - t.Run("fixed peer, http:// URL", func(t *testing.T) { - for _, p := range []string{"", "/", "///"} { - t.Run("w/ path=["+p+"]", func(t *testing.T) { - pps, err := ParseProviderStrings("http://127.0.0.1:5000" + p) - require.NoError(t, err) - request := RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - Providers: pps, - } - ds, err := request.GetDescriptorString() - require.NoError(t, err) - expectedStart := "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n&providers=/ip4/127.0.0.1/tcp/5000/http/p2p/1TunknownX" - require.Equal(t, expectedStart, ds[0:len(expectedStart)]) - }) - } - }) - - t.Run("fixed peer, http:// URL with path err", func(t *testing.T) { - _, err := ParseProviderStrings("http://127.0.0.1:5000/nope") - require.ErrorContains(t, err, "paths not supported") - }) - - t.Run("fixed peer, protocol included", func(t *testing.T) { - pps, err := ParseProviderStrings("/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4+bitswap") - require.NoError(t, err) - require.Equal(t, pps[0].Protocols, []metadata.Protocol{metadata.Bitswap{}}) - request := RetrievalRequest{ - Request: trustlessutils.Request{Root: testCidV1}, - Providers: pps, - } - ds, err := request.GetDescriptorString() - require.NoError(t, err) - expectedStart := "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=all&dups=n&providers=/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4+bitswap" - require.Equal(t, expectedStart, ds[0:len(expectedStart)]) - }) - -} - -func TestProviderStrings(t *testing.T) { - testCases := []struct { - name string - input string - expectMatch string // regex - expectErr string - }{ - { - name: "empty", - input: "", - expectErr: "failed to parse multiaddr", - }, - { - name: "single", - input: "/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg", - expectMatch: "^/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg$", - }, - { - name: "multi", - input: "/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg,/dns4/dag.w3s.link/tcp/443/https/p2p/QmUA9D3H7HeCYsirB3KmPSvZh3dNXMZas6Lwgr4fv1HTTp", - expectMatch: "^/dns/beep.boop.com/tcp/3747/p2p/12D3KooWDXAVxjSTKbHKpNk8mFVQzHdBDvR4kybu582Xd4Zrvagg,/dns4/dag.w3s.link/tcp/443/https/p2p/QmUA9D3H7HeCYsirB3KmPSvZh3dNXMZas6Lwgr4fv1HTTp$", - }, - { - name: "no id", - input: "/ip4/127.0.0.1/tcp/5000/http", - expectMatch: "^/ip4/127.0.0.1/tcp/5000/http/p2p/1TunknownX", - }, - { - name: "http:// form", - input: "http://127.0.0.1:5000", - expectMatch: "^/ip4/127.0.0.1/tcp/5000/http/p2p/1TunknownX", - }, - { - name: "http:// form with path err", - input: "http://127.0.0.1:5000/boop", - expectErr: "invalid provider URL, paths not supported:", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - parsed, err := ParseProviderStrings(tc.input) - if tc.expectErr != "" { - require.Contains(t, err.Error(), tc.expectErr) - return - } else { - require.NoError(t, err) - } - - actual, err := ToProviderString(parsed) - require.NoError(t, err) - - require.Regexp(t, tc.expectMatch, actual) - }) - } -} - -func TestUnknownPeerID(t *testing.T) { - for i := 0; i < 1000; i++ { - p := nextUnknownPeerID() - require.Equal(t, "1TunknownX", p.String()[0:10]) - require.True(t, IsUnknownPeerID(p)) - } -} - -func must[T any](v T, err error) T { - if err != nil { - panic(err) - } - return v -} - -func ptr(i int64) *int64 { - return &i -} diff --git a/pkg/types/types.go b/pkg/types/types.go index b08f18ea..a4ec9fb5 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -7,7 +7,6 @@ import ( "net/url" "time" - "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" "github.com/ipni/go-libipni/maurl" "github.com/ipni/go-libipni/metadata" @@ -73,6 +72,15 @@ func NewRetrievalCandidate(pid peer.ID, addrs []multiaddr.Multiaddr, rootCid cid } } +// Endpoint returns a string representation of the provider's address for logging. +// Prefers multiaddr format (e.g. /ip4/1.2.3.4/tcp/443/https) over peer ID. +func (rc RetrievalCandidate) Endpoint() string { + if len(rc.MinerPeer.Addrs) > 0 { + return rc.MinerPeer.Addrs[0].String() + } + return rc.MinerPeer.ID.String() +} + // ToURL generates a valid HTTP URL from the candidate if possible func (rc RetrievalCandidate) ToURL() (*url.URL, error) { var err error @@ -93,11 +101,6 @@ func (rc RetrievalCandidate) ToURL() (*url.URL, error) { return url, err } -// retrieval task is any task that can be run to produce a result -type RetrievalTask interface { - Run() (*RetrievalStats, error) -} - type Retriever interface { Retrieve(ctx context.Context, request RetrievalRequest, events func(RetrievalEvent)) (*RetrievalStats, error) } @@ -151,22 +154,6 @@ type CandidateRetriever interface { Retrieve(ctx context.Context, request RetrievalRequest, events func(RetrievalEvent)) CandidateRetrieval } -type RetrievalSplitter[T comparable] interface { - SplitCandidates([]RetrievalCandidate) (map[T][]RetrievalCandidate, error) -} - -type CandidateSplitter[T comparable] interface { - SplitRetrievalRequest(ctx context.Context, request RetrievalRequest, events func(RetrievalEvent)) RetrievalSplitter[T] -} - -type AsyncRetrievalSplitter[T comparable] interface { - SplitAsyncCandidates(asyncCandidates InboundAsyncCandidates) (map[T]InboundAsyncCandidates, <-chan error) -} - -type AsyncCandidateSplitter[T comparable] interface { - SplitRetrievalRequest(ctx context.Context, request RetrievalRequest, events func(RetrievalEvent)) AsyncRetrievalSplitter[T] -} - type RetrievalStats struct { StorageProviderId peer.ID RootCid cid.Cid @@ -174,11 +161,10 @@ type RetrievalStats struct { Blocks uint64 Duration time.Duration AverageSpeed uint64 - TotalPayment abi.TokenAmount - NumPayments int - AskPrice abi.TokenAmount TimeToFirstByte time.Duration Selector string + // Providers lists HTTP endpoints that served blocks during retrieval + Providers []string } type RetrievalResult struct { @@ -186,48 +172,6 @@ type RetrievalResult struct { Err error } -var _ RetrievalTask = AsyncRetrievalTask{} - -// AsyncRetrievalTask runs an asynchronous retrieval and returns a result -type AsyncRetrievalTask struct { - Candidates InboundAsyncCandidates - AsyncCandidateRetrieval CandidateRetrieval -} - -// Run executes the asynchronous retrieval task -func (art AsyncRetrievalTask) Run() (*RetrievalStats, error) { - return art.AsyncCandidateRetrieval.RetrieveFromAsyncCandidates(art.Candidates) -} - -var _ RetrievalTask = DeferredErrorTask{} - -// DeferredErrorTask simply reads from an error channel and returns the result as an error -type DeferredErrorTask struct { - Ctx context.Context - ErrChan <-chan error -} - -// Run reads the error channel and returns a result -func (det DeferredErrorTask) Run() (*RetrievalStats, error) { - select { - case <-det.Ctx.Done(): - return nil, det.Ctx.Err() - case err := <-det.ErrChan: - return nil, err - } -} - -type QueueRetrievalsFn func(ctx context.Context, nextRetrievalCall func(RetrievalTask)) - -type RetrievalCoordinator func(context.Context, QueueRetrievalsFn) (*RetrievalStats, error) - -type CoordinationKind string - -const ( - RaceCoordination = "race" - SequentialCoordination = "sequential" -) - type EventCode string const ( @@ -263,8 +207,6 @@ type RetrievalEvent interface { RootCid() cid.Cid } -const BitswapIndentifier = "Bitswap" - // RetrievalEventSubscriber is a function that receives a stream of retrieval // events from all retrievals that are in progress. Various different types // implement the RetrievalEvent interface and may contain additional information