feat: add experimental on-demand pinning #11252
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package config | ||
|
|
||
| import "time" | ||
|
|
||
| const ( | ||
| DefaultOnDemandPinReplicationTarget = 5 | ||
| DefaultOnDemandPinCheckInterval = 10 * time.Minute | ||
| DefaultOnDemandPinUnpinGracePeriod = 24 * time.Hour | ||
| ) | ||
|
|
||
| type OnDemandPinning struct { | ||
|
Member
There was a problem hiding this comment. ℹ️ Could we settle the naming before this ships, and borrow ipfs-cluster's words instead of new ones? "On demand" normally means fetch it when someone asks for it (video on demand). This checker does the opposite: it counts providers and pins when the content is rare. Cluster already has a vocabulary for that idea: Concretely: config section The min/max pair is worth borrowing too, not just the words. Cluster's min is the minimum healthy copies, and its max is the target number of nodes that should pin. The gap between the two is the deadband this checker does not have today (see my note on The This is much easier now than later. The pin name If the current naming is deliberate, I would rather hear your reasoning than push a rename through.
Author
There was a problem hiding this comment. When going forward with ReplicationPinning, the user might ask:
I chose the wording to reflect a "demand to replicate". If a CID is at risk to be lost - only then start pinning. Keeping the word "pinning" is essential to show the similarities. So that is a strength of both names that we should keep. ReplicationPinning tells the user what it is doing, but does not tell the user what it is not doing. We could go with ConditionalPinning to keep the feature extensible for other conditions. Available disk space is such a condition, so it would be transparent that the replication demand is not the only condition. On the other hand, I still think OnDemand pinning reflects the goal of the feature very well (The network needs you to jump in and replicate to protect the data from being lost). "Uncle Sam wants you" vs "IPFS demands you to protect your data."
Author
There was a problem hiding this comment. I am open to follow your intuition here. Feel free to make a decision after considering my argument.
Member
There was a problem hiding this comment. I don't feel strongly about this tbh, just wanted to ensure the name is not a placeholder. If you want to keep "on demand" that is fine with me, just make sure docs / |
||
| // Minimum providers desired in the DHT (excluding self). | ||
| ReplicationTarget OptionalInteger | ||
|
|
||
| // How often the checker evaluates all registered CIDs. | ||
| CheckInterval OptionalDuration | ||
|
|
||
| // How long replication must stay above target before unpinning. | ||
| UnpinGracePeriod OptionalDuration | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,280 @@ | ||
| package pin | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "time" | ||
|
|
||
| cmds "github.com/ipfs/go-ipfs-cmds" | ||
| "github.com/ipfs/kubo/config" | ||
| cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" | ||
| "github.com/ipfs/kubo/core/commands/cmdutils" | ||
| "github.com/ipfs/kubo/ondemandpin" | ||
| ) | ||
|
|
||
| const onDemandLiveOptionName = "live" | ||
|
Author
There was a problem hiding this comment. the ondemand ls command shows the locally stored pin records (CID, pinned status, timestamps). With ---live, it additionally queries the DHT in real-time to report how many providers currently exist
Author
There was a problem hiding this comment. DHT queries are sequential. Doing |
||
|
|
||
| var onDemandPinCmd = &cmds.Command{ | ||
| Helptext: cmds.HelpText{ | ||
| Tagline: "Manage on-demand pins.", | ||
| ShortDescription: ` | ||
| On-demand pins when few DHT providers exist in the routing table; unpins after | ||
| replication stays above target for a grace period. Requires config | ||
| Experimental.OnDemandPinningEnabled. | ||
| `, | ||
| }, | ||
| Subcommands: map[string]*cmds.Command{ | ||
| "add": addOnDemandPinCmd, | ||
| "rm": rmOnDemandPinCmd, | ||
| "ls": listOnDemandPinCmd, | ||
| }, | ||
| } | ||
|
|
||
| type OnDemandPinOutput struct { | ||
| Cid string `json:"Cid"` | ||
| } | ||
|
|
||
| var addOnDemandPinCmd = &cmds.Command{ | ||
| Helptext: cmds.HelpText{ | ||
| Tagline: "Register CIDs for on-demand pinning.", | ||
| ShortDescription: `Registers CID(s) for on-demand pinning; checker pins when needed.`, | ||
| }, | ||
| Arguments: []cmds.Argument{ | ||
| cmds.StringArg("cid", true, true, "CID(s) to register."), | ||
| }, | ||
| Type: OnDemandPinOutput{}, | ||
| Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { | ||
| n, err := cmdenv.GetNode(env) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| cfg, err := n.Repo.Config() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if !cfg.Experimental.OnDemandPinningEnabled { | ||
| return fmt.Errorf("on-demand pinning is not enabled; set Experimental.OnDemandPinningEnabled = true in config") | ||
| } | ||
|
|
||
| store := n.OnDemandPinStore | ||
|
|
||
| api, err := cmdenv.GetApi(env, req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, arg := range req.Arguments { | ||
| p, err := cmdutils.PathOrCidPath(arg) | ||
| if err != nil { | ||
| return fmt.Errorf("invalid CID or path %q: %w", arg, err) | ||
| } | ||
|
|
||
| rp, _, err := api.ResolvePath(req.Context, p) | ||
| if err != nil { | ||
| return fmt.Errorf("resolving %q: %w", arg, err) | ||
| } | ||
| c := rp.RootCid() | ||
|
|
||
| if err := store.Add(req.Context, c); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if checker := n.OnDemandPinChecker; checker != nil { | ||
| checker.Enqueue(c) | ||
| } | ||
|
|
||
| if err := res.Emit(&OnDemandPinOutput{ | ||
| Cid: c.String(), | ||
| }); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| }, | ||
| Encoders: cmds.EncoderMap{ | ||
| cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandPinOutput) error { | ||
| fmt.Fprintf(w, "registered %s for on-demand pinning\n", out.Cid) | ||
| return nil | ||
| }), | ||
| }, | ||
| } | ||
|
|
||
| var rmOnDemandPinCmd = &cmds.Command{ | ||
| Helptext: cmds.HelpText{ | ||
| Tagline: "Remove CIDs from on-demand pinning.", | ||
| ShortDescription: ` | ||
| Removes CID(s) from the registry. Checker-pinned content is unpinned. | ||
|
|
||
| Works when on-demand pinning is disabled, to clear old registrations. | ||
| `, | ||
| }, | ||
| Arguments: []cmds.Argument{ | ||
| cmds.StringArg("cid", true, true, "CID(s) to remove."), | ||
| }, | ||
| Type: OnDemandPinOutput{}, | ||
| Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { | ||
| n, err := cmdenv.GetNode(env) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| store := n.OnDemandPinStore | ||
|
|
||
| api, err := cmdenv.GetApi(env, req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, arg := range req.Arguments { | ||
| p, err := cmdutils.PathOrCidPath(arg) | ||
| if err != nil { | ||
| return fmt.Errorf("invalid CID or path %q: %w", arg, err) | ||
| } | ||
|
|
||
| rp, _, err := api.ResolvePath(req.Context, p) | ||
| if err != nil { | ||
| return fmt.Errorf("resolving %q: %w", arg, err) | ||
| } | ||
| c := rp.RootCid() | ||
|
|
||
| isOurs, err := ondemandpin.PinHasName(req.Context, n.Pinning, c, ondemandpin.OnDemandPinName) | ||
|
Member
There was a problem hiding this comment.
So a user can create their own pin with The order makes it worse: Could |
||
| if err != nil { | ||
| return fmt.Errorf("checking pin state for %s: %w", c, err) | ||
| } | ||
| if isOurs { | ||
| if err := api.Pin().Rm(req.Context, rp); err != nil { | ||
| return fmt.Errorf("unpinning %s: %w", c, err) | ||
| } | ||
| } | ||
|
|
||
| if err := store.Remove(req.Context, c); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if err := res.Emit(&OnDemandPinOutput{ | ||
| Cid: c.String(), | ||
| }); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| }, | ||
| Encoders: cmds.EncoderMap{ | ||
| cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandPinOutput) error { | ||
| fmt.Fprintf(w, "removed %s from on-demand pinning\n", out.Cid) | ||
| return nil | ||
| }), | ||
| }, | ||
| } | ||
|
|
||
| type OnDemandLsOutput struct { | ||
|
Member
There was a problem hiding this comment. General UX nits: for a feature that unpins on its own, The stored The unpin time is knowable, Could A dry-run mode would help too, so an operator can watch what the checker would do for a week before letting it unpin anything. |
||
| Cid string `json:"Cid"` | ||
| PinnedByUs bool `json:"PinnedByUs"` | ||
| Providers *int `json:"Providers,omitempty"` | ||
| LastAboveTarget string `json:"LastAboveTarget,omitempty"` | ||
| CreatedAt string `json:"CreatedAt"` | ||
| } | ||
|
|
||
| var listOnDemandPinCmd = &cmds.Command{ | ||
| Helptext: cmds.HelpText{ | ||
| Tagline: "List on-demand pins.", | ||
| ShortDescription: ` | ||
| Lists CIDs registered for on-demand pinning with their current state. | ||
| Use --live to include real-time provider counts from the DHT. | ||
| `, | ||
| }, | ||
| Arguments: []cmds.Argument{ | ||
| cmds.StringArg("cid", false, true, "Optional CID(s) to filter."), | ||
| }, | ||
| Options: []cmds.Option{ | ||
| cmds.BoolOption(onDemandLiveOptionName, "l", "Perform live provider lookup."), | ||
| }, | ||
| Type: OnDemandLsOutput{}, | ||
| Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { | ||
| n, err := cmdenv.GetNode(env) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| store := n.OnDemandPinStore | ||
|
|
||
| live, _ := req.Options[onDemandLiveOptionName].(bool) | ||
|
|
||
| var globalTarget int | ||
| if live { | ||
| cfg, err := n.Repo.Config() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| globalTarget = int(cfg.OnDemandPinning.ReplicationTarget.WithDefault(config.DefaultOnDemandPinReplicationTarget)) | ||
| } | ||
|
|
||
| var records []ondemandpin.Record | ||
| if len(req.Arguments) > 0 { | ||
| api, err := cmdenv.GetApi(env, req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| for _, arg := range req.Arguments { | ||
| p, err := cmdutils.PathOrCidPath(arg) | ||
| if err != nil { | ||
| return fmt.Errorf("invalid CID or path %q: %w", arg, err) | ||
| } | ||
| rp, _, err := api.ResolvePath(req.Context, p) | ||
| if err != nil { | ||
| return fmt.Errorf("resolving %q: %w", arg, err) | ||
| } | ||
| rec, err := store.Get(req.Context, rp.RootCid()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| records = append(records, *rec) | ||
| } | ||
| } else { | ||
| records, err = store.List(req.Context) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| for _, rec := range records { | ||
| out := OnDemandLsOutput{ | ||
| Cid: rec.Cid.String(), | ||
| PinnedByUs: rec.PinnedByUs, | ||
| CreatedAt: rec.CreatedAt.Format(time.RFC3339), | ||
| } | ||
| if !rec.LastAboveTarget.IsZero() { | ||
| out.LastAboveTarget = rec.LastAboveTarget.Format(time.RFC3339) | ||
| } | ||
|
|
||
| if live && n.Routing != nil { | ||
|
Member
There was a problem hiding this comment. 🤔 hm.. iiuc this also is pretty brittle:
Could it return an error there, or explicitly say that provider counts are unavailable? In general, try to write code that avoids hiding the fact silent failures occured, better to be explicit and/or loud. |
||
| count := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, globalTarget) | ||
| out.Providers = &count | ||
| } | ||
|
|
||
| if err := res.Emit(&out); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| }, | ||
| Encoders: cmds.EncoderMap{ | ||
| cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandLsOutput) error { | ||
| pinState := "not-pinned" | ||
| if out.PinnedByUs { | ||
| pinState = "pinned" | ||
| } | ||
| fmt.Fprintf(w, "%s", out.Cid) | ||
| if out.Providers != nil { | ||
| fmt.Fprintf(w, " providers=%d", *out.Providers) | ||
| } | ||
| fmt.Fprintf(w, " %s created=%s", pinState, out.CreatedAt) | ||
| if out.LastAboveTarget != "" { | ||
| fmt.Fprintf(w, " above-target-since=%s", out.LastAboveTarget) | ||
| } | ||
| fmt.Fprintln(w) | ||
| return nil | ||
| }), | ||
| }, | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
DefaultOnDemandPinUnpinGracePeriodis shorter than how long a DHT provider record stays valid (amino.DefaultProvideValidityis 48h), andCountProviderscannot tell a stale record from a live peer.A provider that goes offline right after the grace timer starts is still returned by the DHT for up to 48h. So this node can stay above
ReplicationTargetfor the full 24h and unpin while it may be the last one still holding the blocks.Could
DefaultOnDemandPinUnpinGracePeriodbe derived fromamino.DefaultProvideValidityinstead of a hardcoded24h? With a grace period longer than the record validity, stale records expire before the period ends, the count drops below target, and the timer resets.config/provide.goalready imports theaminopackage and checksProvide.DHT.Intervalagainst that constant, so the default here would follow upstream if the DHT ever changes it.