feat(gateway): Gateway.FastDirIndexThreshold (#8853)

* fix(core/gateway): option to limit directory size listing

* feat(gw): HTMLDirListingLimit

This is alternative take on the way we limit the HTML listing output.
Instead of a hard cut-off, we list up to HTMLDirListingLimit.
When a directory has more items than HTMLDirListingLimit we show
additional header and footer informing user that only $HTMLDirListingLimit
items are listed. This is a better UX.

* fix: 0 disables Gateway.HTMLDirListingLimit

* refactor: Gateway.FastDirIndexThreshold

see explainer in docs/config.md

* refactor: prealoc slices

* docs: Gateway.FastDirIndexThreshold

* refactor: core/corehttp/gateway_handler.go

ipfs/kubo#8853 (comment)

* docs: apply suggestions from code review

Co-authored-by: Alan Shaw <[email protected]>

Co-authored-by: Marcin Rataj <[email protected]>
Co-authored-by: Alan Shaw <[email protected]>

This commit was moved from ipfs/kubo@25cc85f

gateway/core/corehttp/gateway.go

@@ -16,9 +16,10 @@ import (
 )
 
 type GatewayConfig struct {
-	Headers      map[string][]string
-	Writable     bool
-	PathPrefixes []string
+	Headers               map[string][]string
+	Writable              bool
+	PathPrefixes          []string
+	FastDirIndexThreshold int
 }
 
 // A helper function to clean up a set of headers:
@@ -89,9 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 			}, headers[ACEHeadersName]...))
 
 		var gateway http.Handler = newGatewayHandler(GatewayConfig{
-			Headers:      headers,
-			Writable:     writable,
-			PathPrefixes: cfg.Gateway.PathPrefixes,
+			Headers:               headers,
+			Writable:              writable,
+			PathPrefixes:          cfg.Gateway.PathPrefixes,
+			FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
 		}, api)
 
 		gateway = otelhttp.NewHandler(gateway, "Gateway.Request")

gateway/core/corehttp/gateway_handler_unixfs.go

@@ -18,6 +18,7 @@ import (
 func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
 	ctx, span := tracing.Span(ctx, "Gateway", "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
 	defer span.End()
+
 	// Handling UnixFS
 	dr, err := i.api.Unixfs().Get(ctx, resolvedPath)
 	if err != nil {
@@ -39,6 +40,7 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
 		internalWebError(w, fmt.Errorf("unsupported UnixFS type"))
 		return
 	}
+
 	logger.Debugw("serving unixfs directory", "path", contentPath)
 	i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger)
 }

gateway/core/corehttp/gateway_handler_unixfs_dir.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ipfs/go-ipfs/tracing"
 	path "github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
+	options "github.com/ipfs/interface-go-ipfs-core/options"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -102,36 +103,46 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 		return
 	}
 
+	// Optimization 1:
+	// List children without fetching their root blocks (fast, but no size info)
+	results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
+	if err != nil {
+		internalWebError(w, err)
+		return
+	}
+
 	// storage for directory listing
-	var dirListing []directoryItem
-	dirit := dir.Entries()
-	for dirit.Next() {
-		size := "?"
-		if s, err := dirit.Node().Size(); err == nil {
-			// Size may not be defined/supported. Continue anyways.
-			size = humanize.Bytes(uint64(s))
-		}
+	dirListing := make([]directoryItem, 0, len(results))
 
-		resolved, err := i.api.ResolvePath(ctx, ipath.Join(resolvedPath, dirit.Name()))
-		if err != nil {
+	for link := range results {
+		if link.Err != nil {
 			internalWebError(w, err)
 			return
 		}
-		hash := resolved.Cid().String()
-
-		// See comment above where originalUrlPath is declared.
+		hash := link.Cid.String()
 		di := directoryItem{
-			Size:      size,
-			Name:      dirit.Name(),
-			Path:      gopath.Join(originalUrlPath, dirit.Name()),
+			Size:      "", // no size because we did not fetch child nodes
+			Name:      link.Name,
+			Path:      gopath.Join(originalUrlPath, link.Name),
 			Hash:      hash,
 			ShortHash: shortHash(hash),
 		}
 		dirListing = append(dirListing, di)
 	}
-	if dirit.Err() != nil {
-		internalWebError(w, dirit.Err())
-		return
+
+	// Optimization 2: fetch sizes only for dirs below FastDirIndexThreshold
+	if len(dirListing) < i.config.FastDirIndexThreshold {
+		dirit := dir.Entries()
+		linkNo := 0
+		for dirit.Next() {
+			size := "?"
+			if s, err := dirit.Node().Size(); err == nil {
+				// Size may not be defined/supported. Continue anyways.
+				size = humanize.Bytes(uint64(s))
+			}
+			dirListing[linkNo].Size = size
+			linkNo++
+		}
 	}
 
 	// construct the correct back link
@@ -180,14 +191,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 
 	// See comment above where originalUrlPath is declared.
 	tplData := listingTemplateData{
-		GatewayURL:  gwURL,
-		DNSLink:     dnslink,
-		Listing:     dirListing,
-		Size:        size,
-		Path:        contentPath.String(),
-		Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
-		BackLink:    backLink,
-		Hash:        hash,
+		GatewayURL:            gwURL,
+		DNSLink:               dnslink,
+		Listing:               dirListing,
+		Size:                  size,
+		Path:                  contentPath.String(),
+		Breadcrumbs:           breadcrumbs(contentPath.String(), dnslink),
+		BackLink:              backLink,
+		Hash:                  hash,
+		FastDirIndexThreshold: i.config.FastDirIndexThreshold,
 	}
 
 	logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)

gateway/core/corehttp/gateway_indexPage.go

@@ -12,14 +12,15 @@ import (
 
 // structs for directory listing
 type listingTemplateData struct {
-	GatewayURL  string
-	DNSLink     bool
-	Listing     []directoryItem
-	Size        string
-	Path        string
-	Breadcrumbs []breadcrumb
-	BackLink    string
-	Hash        string
+	GatewayURL            string
+	DNSLink               bool
+	Listing               []directoryItem
+	Size                  string
+	Path                  string
+	Breadcrumbs           []breadcrumb
+	BackLink              string
+	Hash                  string
+	FastDirIndexThreshold int
 }
 
 type directoryItem struct {