From 40f687640b7a846a2ac48d106376cb168443f3cd Mon Sep 17 00:00:00 2001 From: Antonio Navarro Perez Date: Fri, 27 Jan 2023 11:08:02 +0100 Subject: [PATCH] Adjust inbound connection limits depending on memory. Signed-off-by: Antonio Navarro Perez --- core/node/libp2p/rcmgr_defaults.go | 68 +++++++++++++----------------- docs/changelogs/v0.18.md | 48 +++++++++++++++------ docs/libp2p-resource-management.md | 8 ++-- 3 files changed, 70 insertions(+), 54 deletions(-) diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index 4d578a9b6504..dd03f109745a 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -44,17 +44,18 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{ // This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled // createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager. -// The defaults follow the documentation in docs/config.md. +// The defaults follow the documentation in docs/libp2p-resource-management.md. // Any changes in the logic here should be reflected there. func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) { - maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 4) + maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 2) maxMemoryString := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefaultString) maxMemory, err := humanize.ParseBytes(maxMemoryString) if err != nil { return rcmgr.LimitConfig{}, err } - numFD := cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2) + maxMemoryMB := maxMemory / (1024 * 1024) + maxFD := cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2) // We want to see this message on startup, that's why we are using fmt instead of log. fmt.Printf(` @@ -65,65 +66,56 @@ Computing default go-libp2p Resource Manager limits based on: Applying any user-supplied overrides on top. Run 'ipfs swarm limit all' to see the resulting limits. -`, maxMemoryString, numFD) +`, maxMemoryString, maxFD) + + // At least as of 2023-01-25, it's possible to open a connection that + // doesn't ask for any memory usage with the libp2p Resource Manager/Accountant + // (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736). + // As a result, we can't curretly rely on Memory limits to full protect us. + // Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed, + // we take a proxy now of restricting to 1 inbound connection per MB. + // Note: this is more generous than go-libp2p's default autoscaled limits which do + // 64 connections per 1GB + // (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ). + systemConnsInbound := int(1 * maxMemoryMB) scalingLimitConfig := rcmgr.ScalingLimitConfig{ SystemBaseLimit: rcmgr.BaseLimit{ Memory: int64(maxMemory), - FD: int(numFD), + FD: int(maxFD), // By default, we just limit connections on the inbound side. Conns: bigEnough, - ConnsInbound: rcmgr.DefaultLimits.SystemBaseLimit.ConnsInbound, // same as libp2p default + ConnsInbound: systemConnsInbound, ConnsOutbound: bigEnough, // We limit streams since they not only take up memory and CPU. // The Memory limit protects us on the memory side, // but a StreamsInbound limit helps protect against unbound CPU consumption from stream processing. Streams: bigEnough, - StreamsInbound: rcmgr.DefaultLimits.SystemBaseLimit.StreamsInbound, + StreamsInbound: bigEnough, StreamsOutbound: bigEnough, }, - // Most limits don't see an increase because they're already infinite/bigEnough or at their max value. - // The values that should scale based on the amount of memory allocated to libp2p need to increase accordingly. - SystemLimitIncrease: rcmgr.BaseLimitIncrease{ - Memory: 0, - FDFraction: 0, - - Conns: 0, - ConnsInbound: rcmgr.DefaultLimits.SystemLimitIncrease.ConnsInbound, - ConnsOutbound: 0, - - Streams: 0, - StreamsInbound: rcmgr.DefaultLimits.SystemLimitIncrease.StreamsInbound, - StreamsOutbound: 0, - }, + SystemLimitIncrease: noLimitIncrease, + // Transient connections won't cause any memory to accounted for by the resource manager. + // Only established connections do. + // As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened. + // We limit the same values as the System scope, but only allow the Transient scope to take 25% of what is allowed for the System scope. TransientBaseLimit: rcmgr.BaseLimit{ - Memory: rcmgr.DefaultLimits.TransientBaseLimit.Memory, - FD: rcmgr.DefaultLimits.TransientBaseLimit.FD, + Memory: int64(maxMemory / 4), + FD: int(maxFD / 4), Conns: bigEnough, - ConnsInbound: rcmgr.DefaultLimits.TransientBaseLimit.ConnsInbound, + ConnsInbound: systemConnsInbound / 4, ConnsOutbound: bigEnough, Streams: bigEnough, - StreamsInbound: rcmgr.DefaultLimits.TransientBaseLimit.StreamsInbound, + StreamsInbound: bigEnough, StreamsOutbound: bigEnough, }, - TransientLimitIncrease: rcmgr.BaseLimitIncrease{ - Memory: rcmgr.DefaultLimits.TransientLimitIncrease.Memory, - FDFraction: rcmgr.DefaultLimits.TransientLimitIncrease.FDFraction, - - Conns: 0, - ConnsInbound: rcmgr.DefaultLimits.TransientLimitIncrease.ConnsInbound, - ConnsOutbound: 0, - - Streams: 0, - StreamsInbound: rcmgr.DefaultLimits.TransientLimitIncrease.StreamsInbound, - StreamsOutbound: 0, - }, + TransientLimitIncrease: noLimitIncrease, // Lets get out of the way of the allow list functionality. // If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through. @@ -184,7 +176,7 @@ Run 'ipfs swarm limit all' to see the resulting limits. // Whatever limits libp2p has specifically tuned for its protocols/services we'll apply. libp2p.SetDefaultServiceLimits(&scalingLimitConfig) - defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), int(numFD)) + defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), int(maxFD)) // Simple checks to overide autoscaling ensuring limits make sense versus the connmgr values. // There are ways to break this, but this should catch most problems already. diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index 376658fd9876..7614ddb08726 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -2,14 +2,14 @@ ## v0.18.1 -This release includes improvements around Pubsub message deduplication, and more. - +This release includes improvements around Pubsub message deduplication, libp2p resource management, and more. - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - [New default Pubsub.SeenMessagesStrategy](#new-default-pubsubseenmessagesstrategy) + - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -33,11 +33,24 @@ If you prefer the old behavior, which calculates the TTL countdown based on the first time a message is seen, you can set `Pubsub.SeenMessagesStrategy` to `first-seen`. +#### Improving libp2p resource management integration + +This builds on the default protection nodes get against DoS (resource exhaustion) and eclipse attacks +with the [go-libp2p Network Resource Manager/Accountant](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md) +that was fine-tuned in [Kubo 0.18](https://github.com/ipfs/kubo/blob/biglep/resource-manager-example-of-what-want/docs/changelogs/v0.18.md#improving-libp2p-resource-management-integration). + +Adding default hard-limits from the Resource Manager/Accountant after the fact is tricky, +and some additional improvements have been made to improve the [computed defaults](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#computed-default-limits). +As much as possible, the aim is for a user to only think about how much memory they want to bound libp2p to, +and not need to think about translating that to hard numbers for connections, streams, etc. +More updates are likely in future Kubo releases, but with this release: +1. ``System.StreamsInbound`` is no longer bounded directly +2. ``System.ConnsInbound``, ``Transient.Memory``, ``Transiet.ConnsInbound`` have higher default computed values. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors - ## v0.18.0 ### Overview @@ -46,22 +59,33 @@ Below is an outline of all that is in this release, so you get a sense of all th -- [Overview](#overview) -- [๐Ÿ”ฆ Highlights](#-highlights) - - [Content routing](#content-routing) +- [Kubo changelog v0.18](#kubo-changelog-v018) + - [v0.18.1](#v0181) + - [๐Ÿ”ฆ Highlights](#-highlights) + - [New default `Pubsub.SeenMessagesStrategy`](#new-default-pubsubseenmessagesstrategy) + - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration) + - [๐Ÿ“ Changelog](#-changelog) + - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) + - [v0.18.0](#v0180) + - [Overview](#overview) + - [๐Ÿ”ฆ Highlights](#-highlights-1) + - [Content routing](#content-routing) - [Default InterPlanetary Network Indexer](#default-interplanetary-network-indexer) - [Increase provider record republish interval and expiration](#increase-provider-record-republish-interval-and-expiration) - - [Gateways](#gateways) - - [DAG-JSON and DAG-CBOR response formats](#dag-json-and-dag-cbor-response-formats) + - [Gateways](#gateways) + - [(DAG-)JSON and (DAG-)CBOR response formats](#dag-json-and-dag-cbor-response-formats) + - [Example 1: DAG-CBOR and DAG-JSON Conversion on Gateway](#example-1-dag-cbor-and-dag-json-conversion-on-gateway) + - [Example 2: Traversing CBOR DAGs](#example-2-traversing-cbor-dags) + - [Example 3: UnixFS directory listing as JSON](#example-3-unixfs-directory-listing-as-json) - [๐ŸŽ Fast directory listings with DAG sizes](#-fast-directory-listings-with-dag-sizes) - - [QUIC and WebTransport](#quic-and-webtransport) + - [QUIC and WebTransport](#quic-and-webtransport) - [WebTransport enabled by default](#webtransport-enabled-by-default) - [QUIC and WebTransport share a single port](#quic-and-webtransport-share-a-single-port) - [Differentiating QUIC versions](#differentiating-quic-versions) - [QUICv1 and WebTransport config migration](#quicv1-and-webtransport-config-migration) - - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration) -- [๐Ÿ“ Changelog](#-changelog) -- [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) + - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration-1) + - [๐Ÿ“ Changelog](#-changelog-1) + - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors-1) diff --git a/docs/libp2p-resource-management.md b/docs/libp2p-resource-management.md index d6b782da1f92..e2d5d27a80af 100644 --- a/docs/libp2p-resource-management.md +++ b/docs/libp2p-resource-management.md @@ -13,8 +13,10 @@ Good places to start are: ## Table of Contents -- [libp2p Network Resource Manager (`Swarm.ResourceMgr`)](#libp2p-network-resource-manager-smallswarmresourcemgrsmall) +- [libp2p Network Resource Manager (`Swarm.ResourceMgr`)](#libp2p-network-resource-manager-swarmresourcemgr) - [Purpose](#purpose) + - [๐Ÿ™‹ Help! The resource manager is protecting my node but I want to understand more](#-help--the-resource-manager-is-protecting-my-node-but-i-want-to-understand-more) + - [Table of Contents](#table-of-contents) - [Levels of Configuration](#levels-of-configuration) - [Approach](#approach) - [Computed Default Limits](#computed-default-limits) @@ -70,8 +72,7 @@ The reason these scopes are chosen is because: Within these scopes, limits are just set on [memory](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#memory), -[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections), -and [*inbound* streams](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#streams). +[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), and [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections). Limits are set based on the `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` inputs above. There are also some special cases where minimum values are enforced. @@ -89,7 +90,6 @@ These become the [active limits](#how-does-one-see-the-active-limits). While `Swarm.ResourceMgr.Limits` can be edited directly, it is also possible to use `ipfs swarm limit` command to inspect and tweak specific limits at runtime. - To see all resources that are close to hitting their respective limit: ```console