Adjust inbound connection limits depending on memory.

Signed-off-by: Antonio Navarro Perez <[email protected]>
ipfs · Jan 27, 2023 · 40f6876 · 40f6876
1 parent a3c70a1
commit 40f6876
Show file tree

Hide file tree

Showing 3 changed files with 70 additions and 54 deletions.
diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go
@@ -44,17 +44,18 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{
 // This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled
 
 // createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager.
-// The defaults follow the documentation in docs/config.md.
+// The defaults follow the documentation in docs/libp2p-resource-management.md.
 // Any changes in the logic here should be reflected there.
 func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) {
-	maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 4)
+	maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 2)
 	maxMemoryString := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefaultString)
 	maxMemory, err := humanize.ParseBytes(maxMemoryString)
 	if err != nil {
 		return rcmgr.LimitConfig{}, err
 	}
 
-	numFD := cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2)
+	maxMemoryMB := maxMemory / (1024 * 1024)
+	maxFD := cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2)
 
 	// We want to see this message on startup, that's why we are using fmt instead of log.
 	fmt.Printf(`
@@ -65,65 +66,56 @@ Computing default go-libp2p Resource Manager limits based on:
 Applying any user-supplied overrides on top.
 Run 'ipfs swarm limit all' to see the resulting limits.
 
-`, maxMemoryString, numFD)
+`, maxMemoryString, maxFD)
+
+	// At least as of 2023-01-25, it's possible to open a connection that
+	// doesn't ask for any memory usage with the libp2p Resource Manager/Accountant
+	// (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736).
+	// As a result, we can't curretly rely on Memory limits to full protect us.
+	// Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed,
+	// we take a proxy now of restricting to 1 inbound connection per MB.
+	// Note: this is more generous than go-libp2p's default autoscaled limits which do
+	// 64 connections per 1GB
+	// (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ).
+	systemConnsInbound := int(1 * maxMemoryMB)
 
 	scalingLimitConfig := rcmgr.ScalingLimitConfig{
 		SystemBaseLimit: rcmgr.BaseLimit{
 			Memory: int64(maxMemory),
-			FD:     int(numFD),
+			FD:     int(maxFD),
 
 			// By default, we just limit connections on the inbound side.
 			Conns:         bigEnough,
-			ConnsInbound:  rcmgr.DefaultLimits.SystemBaseLimit.ConnsInbound, // same as libp2p default
+			ConnsInbound:  systemConnsInbound,
 			ConnsOutbound: bigEnough,
 
 			// We limit streams since they not only take up memory and CPU.
 			// The Memory limit protects us on the memory side,
 			// but a StreamsInbound limit helps protect against unbound CPU consumption from stream processing.
 			Streams:         bigEnough,
-			StreamsInbound:  rcmgr.DefaultLimits.SystemBaseLimit.StreamsInbound,
+			StreamsInbound:  bigEnough,
 			StreamsOutbound: bigEnough,
 		},
-		// Most limits don't see an increase because they're already infinite/bigEnough or at their max value.
-		// The values that should scale based on the amount of memory allocated to libp2p need to increase accordingly.
-		SystemLimitIncrease: rcmgr.BaseLimitIncrease{
-			Memory:     0,
-			FDFraction: 0,
-
-			Conns:         0,
-			ConnsInbound:  rcmgr.DefaultLimits.SystemLimitIncrease.ConnsInbound,
-			ConnsOutbound: 0,
-
-			Streams:         0,
-			StreamsInbound:  rcmgr.DefaultLimits.SystemLimitIncrease.StreamsInbound,
-			StreamsOutbound: 0,
-		},
+		SystemLimitIncrease: noLimitIncrease,
 
+		// Transient connections won't cause any memory to accounted for by the resource manager.
+		// Only established connections do.
+		// As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened.
+		// We limit the same values as the System scope, but only allow the Transient scope to take 25% of what is allowed for the System scope.
 		TransientBaseLimit: rcmgr.BaseLimit{
-			Memory: rcmgr.DefaultLimits.TransientBaseLimit.Memory,
-			FD:     rcmgr.DefaultLimits.TransientBaseLimit.FD,
+			Memory: int64(maxMemory / 4),
+			FD:     int(maxFD / 4),
 
 			Conns:         bigEnough,
-			ConnsInbound:  rcmgr.DefaultLimits.TransientBaseLimit.ConnsInbound,
+			ConnsInbound:  systemConnsInbound / 4,
 			ConnsOutbound: bigEnough,
 
 			Streams:         bigEnough,
-			StreamsInbound:  rcmgr.DefaultLimits.TransientBaseLimit.StreamsInbound,
+			StreamsInbound:  bigEnough,
 			StreamsOutbound: bigEnough,
 		},
 
-		TransientLimitIncrease: rcmgr.BaseLimitIncrease{
-			Memory:     rcmgr.DefaultLimits.TransientLimitIncrease.Memory,
-			FDFraction: rcmgr.DefaultLimits.TransientLimitIncrease.FDFraction,
-
-			Conns:         0,
-			ConnsInbound:  rcmgr.DefaultLimits.TransientLimitIncrease.ConnsInbound,
-			ConnsOutbound: 0,
-
-			Streams:         0,
-			StreamsInbound:  rcmgr.DefaultLimits.TransientLimitIncrease.StreamsInbound,
-			StreamsOutbound: 0,
-		},
+		TransientLimitIncrease: noLimitIncrease,
 
 		// Lets get out of the way of the allow list functionality.
 		// If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through.
@@ -184,7 +176,7 @@ Run 'ipfs swarm limit all' to see the resulting limits.
 	// Whatever limits libp2p has specifically tuned for its protocols/services we'll apply.
 	libp2p.SetDefaultServiceLimits(&scalingLimitConfig)
 
-	defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), int(numFD))
+	defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), int(maxFD))
 
 	// Simple checks to overide autoscaling ensuring limits make sense versus the connmgr values.
 	// There are ways to break this, but this should catch most problems already.

diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md
@@ -2,14 +2,14 @@
 
 ## v0.18.1
 
-This release includes improvements around Pubsub message deduplication, and more.
-
+This release includes improvements around Pubsub message deduplication, libp2p resource management, and more.
 
 <!-- TOC depthfrom:3 -->
 
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
     - [New default Pubsub.SeenMessagesStrategy](#new-default-pubsubseenmessagesstrategy)
+    - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -33,11 +33,24 @@ If you prefer the old behavior, which calculates the TTL countdown based on the
 first time a message is seen, you can set `Pubsub.SeenMessagesStrategy` to
 `first-seen`.
 
+#### Improving libp2p resource management integration
+
+This builds on the default protection nodes get against DoS (resource exhaustion) and eclipse attacks
+with the [go-libp2p Network Resource Manager/Accountant](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md)
+that was fine-tuned in [Kubo 0.18](https://github.com/ipfs/kubo/blob/biglep/resource-manager-example-of-what-want/docs/changelogs/v0.18.md#improving-libp2p-resource-management-integration).
+
+Adding default hard-limits from the Resource Manager/Accountant after the fact is tricky,
+and some additional improvements have been made to improve the [computed defaults](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#computed-default-limits).
+As much as possible, the aim is for a user to only think about how much memory they want to bound libp2p to, 
+and not need to think about translating that to hard numbers for connections, streams, etc.
+More updates are likely in future Kubo releases, but with this release: 
+1. ``System.StreamsInbound`` is no longer bounded directly
+2. ``System.ConnsInbound``, ``Transient.Memory``, ``Transiet.ConnsInbound`` have higher default computed values.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors
 
-
 ## v0.18.0
 
 ### Overview
@@ -46,22 +59,33 @@ Below is an outline of all that is in this release, so you get a sense of all th
 
 <!-- TOC depthfrom:3 -->
 
-- [Overview](#overview)
-- [🔦 Highlights](#-highlights)
-    - [Content routing](#content-routing)
+- [Kubo changelog v0.18](#kubo-changelog-v018)
+  - [v0.18.1](#v0181)
+    - [🔦 Highlights](#-highlights)
+      - [New default `Pubsub.SeenMessagesStrategy`](#new-default-pubsubseenmessagesstrategy)
+      - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration)
+    - [📝 Changelog](#-changelog)
+    - [👨‍👩‍👧‍👦 Contributors](#-contributors)
+  - [v0.18.0](#v0180)
+    - [Overview](#overview)
+    - [🔦 Highlights](#-highlights-1)
+      - [Content routing](#content-routing)
         - [Default InterPlanetary Network Indexer](#default-interplanetary-network-indexer)
         - [Increase provider record republish interval and expiration](#increase-provider-record-republish-interval-and-expiration)
-    - [Gateways](#gateways)
-        - [DAG-JSON and DAG-CBOR response formats](#dag-json-and-dag-cbor-response-formats)
+      - [Gateways](#gateways)
+        - [(DAG-)JSON and (DAG-)CBOR response formats](#dag-json-and-dag-cbor-response-formats)
+          - [Example 1: DAG-CBOR and DAG-JSON Conversion on Gateway](#example-1-dag-cbor-and-dag-json-conversion-on-gateway)
+          - [Example 2: Traversing CBOR DAGs](#example-2-traversing-cbor-dags)
+          - [Example 3: UnixFS directory listing as JSON](#example-3-unixfs-directory-listing-as-json)
         - [🐎 Fast directory listings with DAG sizes](#-fast-directory-listings-with-dag-sizes)
-    - [QUIC and WebTransport](#quic-and-webtransport)
+      - [QUIC and WebTransport](#quic-and-webtransport)
         - [WebTransport enabled by default](#webtransport-enabled-by-default)
         - [QUIC and WebTransport share a single port](#quic-and-webtransport-share-a-single-port)
         - [Differentiating QUIC versions](#differentiating-quic-versions)
         - [QUICv1 and WebTransport config migration](#quicv1-and-webtransport-config-migration)
-    - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration)
-- [📝 Changelog](#-changelog)
-- [👨‍👩‍👧‍👦 Contributors](#-contributors)
+      - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration-1)
+    - [📝 Changelog](#-changelog-1)
+    - [👨‍👩‍👧‍👦 Contributors](#-contributors-1)
 
 <!-- /TOC -->
 

diff --git a/docs/libp2p-resource-management.md b/docs/libp2p-resource-management.md
@@ -13,8 +13,10 @@ Good places to start are:
 
 ## Table of Contents
 
-- [libp2p Network Resource Manager <small>(`Swarm.ResourceMgr`)</small>](#libp2p-network-resource-manager-smallswarmresourcemgrsmall)
+- [libp2p Network Resource Manager (`Swarm.ResourceMgr`)](#libp2p-network-resource-manager-swarmresourcemgr)
   - [Purpose](#purpose)
+  - [🙋 Help!  The resource manager is protecting my node but I want to understand more](#-help--the-resource-manager-is-protecting-my-node-but-i-want-to-understand-more)
+  - [Table of Contents](#table-of-contents)
   - [Levels of Configuration](#levels-of-configuration)
     - [Approach](#approach)
     - [Computed Default Limits](#computed-default-limits)
@@ -70,8 +72,7 @@ The reason these scopes are chosen is because:
 
 Within these scopes, limits are just set on
 [memory](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#memory),
-[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections),
-and [*inbound* streams](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#streams).
+[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), and [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections).
 Limits are set based on the `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` inputs above.
 
 There are also some special cases where minimum values are enforced.
@@ -89,7 +90,6 @@ These become the [active limits](#how-does-one-see-the-active-limits).
 
 While `Swarm.ResourceMgr.Limits` can be edited directly, it is also possible to use `ipfs swarm limit` command to inspect and tweak specific limits at runtime.
 
-
 To see all resources that are close to hitting their respective limit:
 
 ```console