diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 1e03a8264a8e..f43f1da242aa 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -425,6 +425,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment case routingOptionNoneKwd: ncfg.Routing = libp2p.NilRouterOption case routingOptionCustomKwd: + if cfg.Routing.AcceleratedDHTClient { + return fmt.Errorf("Routing.AcceleratedDHTClient option is set even tho Routing.Type is custom, using custom .AcceleratedDHTClient needs to be set on DHT routers individually") + } ncfg.Routing = libp2p.ConstructDelegatedRouting( cfg.Routing.Routers, cfg.Routing.Methods, diff --git a/config/experiments.go b/config/experiments.go index 072dcd0ddd62..f5ecf4be6292 100644 --- a/config/experiments.go +++ b/config/experiments.go @@ -8,7 +8,7 @@ type Experiments struct { Libp2pStreamMounting bool P2pHttpProxy bool //nolint StrategicProviding bool - AcceleratedDHTClient bool + AcceleratedDHTClient experimentalAcceleratedDHTClient `json:",omitempty"` OptimisticProvide bool OptimisticProvideJobsPoolSize int } diff --git a/config/routing.go b/config/routing.go index 1210bb3cecc3..7f5e48aa262b 100644 --- a/config/routing.go +++ b/config/routing.go @@ -15,6 +15,8 @@ type Routing struct { // When "custom" is set, user-provided Routing.Routers is used. Type *OptionalString `json:",omitempty"` + AcceleratedDHTClient bool + Routers Routers Methods Methods diff --git a/config/types.go b/config/types.go index 3a0d4f4b3b2d..a781f023af91 100644 --- a/config/types.go +++ b/config/types.go @@ -438,3 +438,27 @@ func (swarmLimits) UnmarshalJSON(b []byte) error { } } } + +type experimentalAcceleratedDHTClient struct{} + +var _ json.Unmarshaler = experimentalAcceleratedDHTClient{} + +func (experimentalAcceleratedDHTClient) UnmarshalJSON(b []byte) error { + d := json.NewDecoder(bytes.NewReader(b)) + for { + switch tok, err := d.Token(); err { + case io.EOF: + return nil + case nil: + switch tok { + case json.Delim('{'), json.Delim('}'): + // accept empty objects + continue + } + //nolint + return fmt.Errorf("The Experimental.AcceleratedDHTClient key has been moved to Routing.AcceleratedDHTClient in Kubo 0.21, please use this new key and remove the old one.") + default: + return err + } + } +} diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index f93f73cb4b8b..6ee51e516fa0 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -7,10 +7,10 @@ import ( "time" humanize "github.com/dustin/go-humanize" + "github.com/ipfs/boxo/provider" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" - - "github.com/ipfs/boxo/provider/batched" + "golang.org/x/exp/constraints" ) var statProvideCmd = &cmds.Command{ @@ -34,12 +34,7 @@ This interface is not stable and may change from release to release. return ErrNotOnline } - sys, ok := nd.Provider.(*batched.BatchProvidingSystem) - if !ok { - return fmt.Errorf("can only return stats if Experimental.AcceleratedDHTClient is enabled") - } - - stats, err := sys.Stat(req.Context) + stats, err := nd.Provider.Stat() if err != nil { return err } @@ -51,7 +46,7 @@ This interface is not stable and may change from release to release. return nil }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *batched.BatchedProviderStats) error { + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *provider.ReproviderStats) error { wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) defer wtr.Flush() @@ -62,14 +57,14 @@ This interface is not stable and may change from release to release. return nil }), }, - Type: batched.BatchedProviderStats{}, + Type: provider.ReproviderStats{}, } func humanDuration(val time.Duration) string { return val.Truncate(time.Microsecond).String() } -func humanNumber(n int) string { +func humanNumber[T constraints.Float | constraints.Integer](n T) string { nf := float64(n) str := humanSI(nf, 0) fullStr := humanFull(nf, 0) diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index 6c97800a3384..5a7e321a9c6e 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -238,7 +238,7 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e return nil, fmt.Errorf("error constructing namesys: %w", err) } - subAPI.provider = provider.NewOfflineProvider() + subAPI.provider = provider.NewNoopProvider() subAPI.peerstore = nil subAPI.peerHost = nil diff --git a/core/node/groups.go b/core/node/groups.go index 5486788358cc..a626a5cae771 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -304,9 +304,9 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part LibP2P(bcfg, cfg, userResourceOverrides), OnlineProviders( cfg.Experimental.StrategicProviding, - cfg.Experimental.AcceleratedDHTClient, cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), + cfg.Routing.AcceleratedDHTClient, ), ) } @@ -320,12 +320,7 @@ func Offline(cfg *config.Config) fx.Option { fx.Provide(libp2p.Routing), fx.Provide(libp2p.ContentRouting), fx.Provide(libp2p.OfflineRouting), - OfflineProviders( - cfg.Experimental.StrategicProviding, - cfg.Experimental.AcceleratedDHTClient, - cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), - cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), - ), + OfflineProviders(), ) } diff --git a/core/node/libp2p/routing.go b/core/node/libp2p/routing.go index 2e356e447f5d..0b642ed8ce18 100644 --- a/core/node/libp2p/routing.go +++ b/core/node/libp2p/routing.go @@ -90,7 +90,7 @@ func BaseRouting(cfg *config.Config) interface{} { } } - if dualDHT != nil && cfg.Experimental.AcceleratedDHTClient { + if dualDHT != nil && cfg.Routing.AcceleratedDHTClient { cfg, err := in.Repo.Config() if err != nil { return out, err diff --git a/core/node/provider.go b/core/node/provider.go index 58e74bdb3f89..215e0adac790 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -5,145 +5,158 @@ import ( "fmt" "time" + "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/fetcher" pin "github.com/ipfs/boxo/pinning/pinner" provider "github.com/ipfs/boxo/provider" - "github.com/ipfs/boxo/provider/batched" - q "github.com/ipfs/boxo/provider/queue" - "github.com/ipfs/boxo/provider/simple" - "go.uber.org/fx" - - "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" + "go.uber.org/fx" ) -// SIMPLE - -// ProviderQueue creates new datastore backed provider queue -func ProviderQueue(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo) (*q.Queue, error) { - return q.NewQueue(helpers.LifecycleCtx(mctx, lc), "provider-v1", repo.Datastore()) -} - -// SimpleProvider creates new record provider -func SimpleProvider(mctx helpers.MetricsCtx, lc fx.Lifecycle, queue *q.Queue, rt irouting.ProvideManyRouter) provider.Provider { - return simple.NewProvider(helpers.LifecycleCtx(mctx, lc), queue, rt) -} - -// SimpleReprovider creates new reprovider -func SimpleReprovider(reproviderInterval time.Duration) interface{} { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, rt irouting.ProvideManyRouter, keyProvider simple.KeyChanFunc) (provider.Reprovider, error) { - return simple.NewReprovider(helpers.LifecycleCtx(mctx, lc), reproviderInterval, rt, keyProvider), nil - } -} - -// SimpleProviderSys creates new provider system -func SimpleProviderSys(isOnline bool) interface{} { - return func(lc fx.Lifecycle, p provider.Provider, r provider.Reprovider) provider.System { - sys := provider.NewSystem(p, r) - - if isOnline { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - sys.Run() - return nil - }, - OnStop: func(ctx context.Context) error { - return sys.Close() - }, - }) +func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option { + const magicThroughputReportCount = 128 + return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) { + opts := []provider.Option{ + provider.Online(cr), + provider.ReproviderInterval(reprovideInterval), + provider.KeyProvider(keyProvider), } - - return sys - } -} - -// BatchedProviderSys creates new provider system -func BatchedProviderSys(isOnline bool, reprovideInterval time.Duration) interface{} { - return func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, q *q.Queue, keyProvider simple.KeyChanFunc, repo repo.Repo) (provider.System, error) { - sys, err := batched.New(cr, q, - batched.ReproviderInterval(reprovideInterval), - batched.Datastore(repo.Datastore()), - batched.KeyProvider(keyProvider)) + if !acceleratedDHTClient { + // The estimation kinda suck if you are running with accelerated DHT client, + // given this message is just trying to push people to use the acceleratedDHTClient + // let's not report on through if it's in use + opts = append(opts, + provider.ThroughputReport(func(reprovide bool, complete bool, keysProvided uint, duration time.Duration) bool { + avgProvideSpeed := duration / time.Duration(keysProvided) + count := uint64(keysProvided) + + if !reprovide || !complete { + // We don't know how many CIDs we have to provide, try to fetch it from the blockstore. + // But don't try for too long as this might be very expensive if you have a huge datastore. + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() + + // FIXME: I want a running counter of blocks so size of blockstore can be an O(1) lookup. + ch, err := bs.AllKeysChan(ctx) + if err != nil { + logger.Errorf("fetching AllKeysChain in provider ThroughputReport: %v", err) + return false + } + count = 0 + countLoop: + for { + select { + case _, ok := <-ch: + if !ok { + break countLoop + } + count++ + case <-ctx.Done(): + // really big blockstore mode + + // how many blocks would be in a 10TiB blockstore with 128KiB blocks. + const probableBigBlockstore = (10 * 1024 * 1024 * 1024 * 1024) / (128 * 1024) + // How long per block that lasts us. + expectedProvideSpeed := reprovideInterval / probableBigBlockstore + if avgProvideSpeed > expectedProvideSpeed { + logger.Errorf(` +๐Ÿ””๐Ÿ””๐Ÿ”” YOU MAY BE FALLING BEHIND DHT REPROVIDES! ๐Ÿ””๐Ÿ””๐Ÿ”” + +โš ๏ธ Your system might be struggling to keep up with DHT reprovides! +This means your content could partially or completely inaccessible on the network. +We observed that you recently provided %d keys at an average rate of %v per key. + +๐Ÿ•‘ An attempt to estimate your blockstore size timed out after 5 minutes, +implying your blockstore might be exceedingly large. Assuming a considerable +size of 10TiB, it would take %v to provide the complete set. + +โฐ The total provide time needs to stay under your reprovide interval (%v) to prevent falling behind! + +๐Ÿ’ก Consider enabling the Accelerated DHT to enhance your system performance. See: +https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient`, + keysProvided, avgProvideSpeed, avgProvideSpeed*probableBigBlockstore, reprovideInterval) + return false + } + } + } + } + + // How long per block that lasts us. + expectedProvideSpeed := reprovideInterval / time.Duration(count) + if avgProvideSpeed > expectedProvideSpeed { + // FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore. + logger.Errorf(` +๐Ÿ””๐Ÿ””๐Ÿ”” YOU ARE FALLING BEHIND DHT REPROVIDES! ๐Ÿ””๐Ÿ””๐Ÿ”” + +โš ๏ธ Your system is struggling to keep up with DHT reprovides! +This means your content could partially or completely inaccessible on the network. +We observed that you recently provided %d keys at an average rate of %v per key. + +๐Ÿ’พ Your total CID count is ~%d which would total at %v reprovide process. + +โฐ The total provide time needs to stay under your reprovide interval (%v) to prevent falling behind! + +๐Ÿ’ก Consider enabling the Accelerated DHT to enhance your reprovide throughput. See: +https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient`, + keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count), reprovideInterval) + } + return false + }, magicThroughputReportCount)) + } + sys, err := provider.New(repo.Datastore(), opts...) if err != nil { return nil, err } - if isOnline { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - sys.Run() - return nil - }, - OnStop: func(ctx context.Context) error { - return sys.Close() - }, - }) - } + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return sys.Close() + }, + }) return sys, nil - } + }) } // ONLINE/OFFLINE // OnlineProviders groups units managing provider routing records online -func OnlineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval time.Duration) fx.Option { - if useStrategicProviding { - return fx.Provide(provider.NewOfflineProvider) - } - - return fx.Options( - SimpleProviders(reprovideStrategy, reprovideInterval), - maybeProvide(SimpleProviderSys(true), !useBatchedProviding), - maybeProvide(BatchedProviderSys(true, reprovideInterval), useBatchedProviding), - ) -} - -// OfflineProviders groups units managing provider routing records offline -func OfflineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval time.Duration) fx.Option { +func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option { if useStrategicProviding { - return fx.Provide(provider.NewOfflineProvider) + return OfflineProviders() } - return fx.Options( - SimpleProviders(reprovideStrategy, reprovideInterval), - maybeProvide(SimpleProviderSys(false), true), - //maybeProvide(BatchedProviderSys(false, reprovideInterval), useBatchedProviding), - ) -} - -// SimpleProviders creates the simple provider/reprovider dependencies -func SimpleProviders(reprovideStrategy string, reproviderInterval time.Duration) fx.Option { var keyProvider fx.Option switch reprovideStrategy { - case "all": - fallthrough - case "": - keyProvider = fx.Provide(simple.NewBlockstoreProvider) + case "all", "": + keyProvider = fx.Provide(provider.NewBlockstoreProvider) case "roots": keyProvider = fx.Provide(pinnedProviderStrategy(true)) case "pinned": keyProvider = fx.Provide(pinnedProviderStrategy(false)) default: - return fx.Error(fmt.Errorf("unknown reprovider strategy '%s'", reprovideStrategy)) + return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy)) } return fx.Options( - fx.Provide(ProviderQueue), - fx.Provide(SimpleProvider), keyProvider, - fx.Provide(SimpleReprovider(reproviderInterval)), + ProviderSys(reprovideInterval, acceleratedDHTClient), ) } +// OfflineProviders groups units managing provider routing records offline +func OfflineProviders() fx.Option { + return fx.Provide(provider.NewNoopProvider) +} + func pinnedProviderStrategy(onlyRoots bool) interface{} { type input struct { fx.In Pinner pin.Pinner IPLDFetcher fetcher.Factory `name:"ipldFetcher"` } - return func(in input) simple.KeyChanFunc { - return simple.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher) + return func(in input) provider.KeyChanFunc { + return provider.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher) } } diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index dccdb505d9e4..82d016ab38ac 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -10,6 +10,7 @@ - [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag) - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) + - [Accelerated DHT Client is no longer experimental](#--empty-repo-is-now-the-default) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -74,7 +75,7 @@ for Kubo `v0.21`. In this release, we improved the HTML templates of our HTTP gateway: -1. You can now preview the contents of a DAG-CBOR and DAG-JSON document from your browser, as well as follow any IPLD Links ([CBOR Tag 42](https://github.com/ipld/cid-cbor/)) contained within them. +1. You can now preview the contents of a DAG-CBOR and DAG-JSON document from your browser, as well as follow any IPLD Links ([CBOR Tag 42](https://github.com/ipld/cid-cbor/)) contained within them. 2. The HTML directory listings now contain [updated, higher-definition icons](https://user-images.githubusercontent.com/5447088/241224419-5385793a-d3bb-40aa-8cb0-0382b5bc56a0.png). 3. On gateway error, instead of a plain text error message, web browsers will now get a friendly HTML response with more details regarding the problem. @@ -84,6 +85,16 @@ HTML responses are returned when request's `Accept` header includes `text/html`. | ---- | ---- | | ![DAG-CBOR Preview](https://github.com/ipfs/boxo/assets/5447088/973f05d1-5731-4469-9da5-d1d776891899) | ![Error Page](https://github.com/ipfs/boxo/assets/5447088/14c453df-adbc-4634-b038-133121914550) | +#### Accelerated DHT Client is no longer experimental + +The [accelerated DHT client](docs/config.md#routingaccelerateddhtclient) is now the main recommended solution for users who +are hosting lots of data. + +The `Experimental.AcceleratedDHTClient` flag moved to `[Routing.AcceleratedDHTClient](docs/config.md#routingaccelerateddhtclient)`. A repo migration has been added. + +A new tracker estimates the providing speed and warns users if they +should be using AcceleratedDHTClient because they are falling behind. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index bf2b7750c641..10994f68b881 100644 --- a/docs/config.md +++ b/docs/config.md @@ -110,6 +110,7 @@ config file at runtime. - [`Reprovider.Strategy`](#reproviderstrategy) - [`Routing`](#routing) - [`Routing.Type`](#routingtype) + - [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient) - [`Routing.Routers`](#routingrouters) - [`Routing.Routers: Type`](#routingrouters-type) - [`Routing.Routers: Parameters`](#routingrouters-parameters) @@ -1348,7 +1349,7 @@ Type: `array[peering]` ### `Reprovider.Interval` Sets the time between rounds of reproviding local content to the routing -system. +system. - If unset, it uses the implicit safe default. - If set to the value `"0"` it will disable content reproviding. @@ -1423,6 +1424,52 @@ Default: `auto` (DHT + IPNI) Type: `optionalString` (`null`/missing means the default) + +### `Routing.AcceleratedDHTClient` + +Utilizes an alternative DHT client using a Full-Routing-Table strategy, it will +every hour do a complete scan of the DHT and record all nodes found. +Then when a lookup is tried instead of having to go through multiple Kad hops it +is able to find the 20 final nodes by looking up the recorded network table. + +This means you trade off cpu and memory due to the extra periodic scans and more +records to keep. However the latency of individual operations should be ~10x faster +and the provide throughput up to 6 millions times faster. + +This is not compatible with `Routing.Type` `custom`. If you are using composable routers +you can configure this individualy on each router. + +When it is enabled: +- DHT operations (reads and writes) should complete much faster giving lower latency +- The provider will now use a keyspace sweeping mode allowing to keep alive + CID sets that are multiple orders of magnitude bigger alive. +- The standard Bucket-Routing-Table DHT will still run for the DHT server if a + mode that enables the DHT server is used. This means the classical routing + table will still be used to answer other nodes, this is because this is critical + to get right in order to not harm the network. +- The operations `ipfs stats dht` will default to showing information about the accelerated DHT client + +**Caveats:** +1. Running the accelerated client likely will result in more resource consumption (connections, RAM, CPU, bandwidth) + - Users that are limited in the number of parallel connections their machines/networks can perform will likely suffer + - The resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly done in rounds + - Users who previously had a lot of content but were unable to advertise it on the network will see an increase in + egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data + entering your node that you don't want to advertise, then consider using [Reprovider Strategies](#reproviderstrategy) + to reduce the number of CIDs that you are reproviding. Similarly, if you are running a node that deals mostly with + short-lived temporary data (e.g. you use a separate node for ingesting data then for storing and serving it) then + you may benefit from using [Strategic Providing](experimental-features.md#strategic-providing) to prevent advertising + of data that you ultimately will not have. +2. Currently, the DHT is not usable for queries for the first 5-10 minutes of operation as the routing table is being +prepared. This means operations like searching the DHT for particular peers or content will not work initially. + - You can see if the DHT has been initially populated by running `ipfs stats dht` +3. Currently, the accelerated DHT client is not compatible with LAN-based DHTs and will not perform operations against +them + +Default: `false` + +Type: `bool` (missing means `false`) + ### `Routing.Routers` **EXPERIMENTAL: `Routing.Routers` configuration may change in future release** @@ -1465,7 +1512,7 @@ HTTP: DHT: - `"Mode"`: Mode used by the DHT. Possible values: "server", "client", "auto" - - `"AcceleratedDHTClient"`: Set to `true` if you want to use the experimentalDHT. + - `"AcceleratedDHTClient"`: Set to `true` if you want to use the acceleratedDHT. - `"PublicIPNetwork"`: Set to `true` to create a `WAN` DHT. Set to `false` to create a `LAN` DHT. Parallel: diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 4ce4275acb89..76d07ebe2028 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 + github.com/ipfs/boxo v0.8.2-0.20230601114506-0ff6929cc9a1 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 @@ -21,7 +21,6 @@ require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 406780e47d90..4f9eca90042a 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -74,8 +74,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -321,8 +319,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 h1:jkmzkt/eRxC+tAOgUYOoQh50Bvfnun/Dy3n72tRSkmo= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230601114506-0ff6929cc9a1 h1:i6SonaZANJjfU+vlM1vQ8P/1OQyMZK2dAbFL+zfV/c4= +github.com/ipfs/boxo v0.8.2-0.20230601114506-0ff6929cc9a1/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 83a5fdf7b863..07f7f30f5e83 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -513,7 +513,7 @@ ipfs config --json Experimental.StrategicProviding true - [ ] provide roots - [ ] provide all - [ ] provide strategic - + ## GraphSync ### State @@ -546,59 +546,6 @@ Stable, enabled by default [Noise](https://github.com/libp2p/specs/tree/master/noise) libp2p transport based on the [Noise Protocol Framework](https://noiseprotocol.org/noise.html). While TLS remains the default transport in Kubo, Noise is easier to implement and is thus the "interop" transport between IPFS and libp2p implementations. -## Accelerated DHT Client - -### In Version - -0.9.0 - -### State - -Experimental, default-disabled. - -Utilizes an alternative DHT client that searches for and maintains more information about the network -in exchange for being more performant. - -When it is enabled: -- DHT operations should complete much faster than with it disabled -- A batching reprovider system will be enabled which takes advantage of some properties of the experimental client to - very efficiently put provider records into the network -- The standard DHT client (and server if enabled) are run alongside the alternative client -- The operations `ipfs stats dht` and `ipfs stats provide` will have different outputs - - `ipfs stats provide` only works when the accelerated DHT client is enabled and shows various statistics regarding - the provider/reprovider system - - `ipfs stats dht` will default to showing information about the new client - -**Caveats:** -1. Running the experimental client likely will result in more resource consumption (connections, RAM, CPU, bandwidth) - - Users that are limited in the number of parallel connections their machines/networks can perform will likely suffer - - Currently, the resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly - done in rounds - - Users who previously had a lot of content but were unable to advertise it on the network will see an increase in - egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data - entering your node that you don't want to advertise consider using [Reprovider Strategies](config.md#reproviderstrategy) - to reduce the number of CIDs that you are reproviding. Similarly, if you are running a node that deals mostly with - short-lived temporary data (e.g. you use a separate node for ingesting data then for storing and serving it) then - you may benefit from using [Strategic Providing](#strategic-providing) to prevent advertising of data that you - ultimately will not have. -2. Currently, the DHT is not usable for queries for the first 5-10 minutes of operation as the routing table is being -prepared. This means operations like searching the DHT for particular peers or content will not work - - You can see if the DHT has been initially populated by running `ipfs stats dht` -3. Currently, the accelerated DHT client is not compatible with LAN-based DHTs and will not perform operations against -them - -### How to enable - -``` -ipfs config --json Experimental.AcceleratedDHTClient true -``` - -### Road to being a real feature - -- [ ] Needs more people to use and report on how well it works -- [ ] Should be usable for queries (even if slower/less efficient) shortly after startup -- [ ] Should be usable with non-WAN DHTs - ## Optimistic Provide ### In Version @@ -640,7 +587,7 @@ than the classic client. size estimation available the client will transparently fall back to the classic approach. 2. The chosen peers to store the provider records might not be the actual closest ones. Measurements showed that this is not a problem. -3. The optimistic provide process returns already after 15 out of the 20 provider records were stored with peers. The +3. The optimistic provide process returns already after 15 out of the 20 provider records were stored with peers. The reasoning here is that one out of the remaining 5 peers are very likely to time out and delay the whole process. To limit the number of in-flight async requests there is the second `OptimisticProvideJobsPoolSize` setting. Currently, this is set to 60. This means that at most 60 parallel background requests are allowed to be in-flight. If this diff --git a/go.mod b/go.mod index 5e980dc2bcf6..1d49bd09c92e 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 + github.com/ipfs/boxo v0.8.2-0.20230601114506-0ff6929cc9a1 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 @@ -82,6 +82,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.9.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/mod v0.10.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.8.0 @@ -93,7 +94,6 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect @@ -215,7 +215,6 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/term v0.8.0 // indirect diff --git a/go.sum b/go.sum index 5e3e2c99af98..aa06b6387981 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -356,8 +354,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 h1:jkmzkt/eRxC+tAOgUYOoQh50Bvfnun/Dy3n72tRSkmo= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230601114506-0ff6929cc9a1 h1:i6SonaZANJjfU+vlM1vQ8P/1OQyMZK2dAbFL+zfV/c4= +github.com/ipfs/boxo v0.8.2-0.20230601114506-0ff6929cc9a1/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index cb8de08c1deb..3c4a709c1955 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -37,7 +37,7 @@ const LockFile = "repo.lock" var log = logging.Logger("fsrepo") // RepoVersion is the version number that we are currently expecting to see -var RepoVersion = 13 +var RepoVersion = 14 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md Sorry for the inconvenience. In the future, these will run automatically.` diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index c174b5e77751..1dc4d0345c78 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -11,7 +11,7 @@ import ( const ( // Current distribution to fetch migrations from - CurrentIpfsDist = "/ipfs/Qmf4yftD4LuMo8JMNPqqw3BtUwYd2VkXMiAThuPE6usrbQ" // fs-repo-12-to-13 v1.0.0 + CurrentIpfsDist = "/ipfs/QmYerugGRCZWA8yQMKDsd9daEVXUR3C5nuw3VXuX1mggHa" // fs-repo-13-to-14 v1.0.0 // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.tech"