Include zpages data in flare (#28927)

LICENSE-3rdparty.csv

@@ -275,7 +275,6 @@ core,github.com/ProtonMail/go-crypto/openpgp/internal/ecc,BSD-3-Clause,Copyright
 core,github.com/ProtonMail/go-crypto/openpgp/internal/encoding,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,github.com/ProtonMail/go-crypto/openpgp/packet,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,github.com/ProtonMail/go-crypto/openpgp/s2k,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
-core,github.com/PuerkitoBio/goquery,BSD-3-Clause,"Copyright (c) 2012-2021, Martin Angers & Contributors"
 core,github.com/Showmax/go-fqdn,Apache-2.0,Copyright since 2015 Showmax s.r.o
 core,github.com/StackExchange/wmi,MIT,Copyright (c) 2013 Stack Exchange
 core,github.com/VividCortex/ewma,MIT,"Copyright (c) 2013 VividCortex | Copyright (c) 2013 VividCortex, Inc. All rights reserved"
@@ -290,10 +289,6 @@ core,github.com/alecthomas/participle/v2,MIT,Copyright (C) 2017 Alec Thomas | Co
 core,github.com/alecthomas/participle/v2/lexer,MIT,Copyright (C) 2017 Alec Thomas | Copyright (C) 2017-2022 Alec Thomas
 core,github.com/alecthomas/units,MIT,Copyright (C) 2014 Alec Thomas
 core,github.com/anchore/go-struct-converter,Apache-2.0,"Copyright (c) 2022-2023 Anchore, Inc."
-core,github.com/andybalholm/cascadia,BSD-2-Clause,Copyright (c) 2011 Andy Balholm. All rights reserved
-core,github.com/antchfx/htmlquery,MIT,Copyright (c) 2016 Zheng Chun
-core,github.com/antchfx/xmlquery,MIT,Copyright (c) 2016 Zheng Chun
-core,github.com/antchfx/xpath,MIT,Copyright (c) 2016 Zheng Chun
 core,github.com/antlr/antlr4/runtime/Go/antlr/v4,BSD-3-Clause,Copyright 2021 The ANTLR Project
 core,github.com/apache/thrift/lib/go/thrift,Apache-2.0,"Copyright (C) 2006 - 2019, The Apache Software Foundation | Copyright (c) 2006- Facebook | Copyright (c) 2006-2008 Alexander Chemeris | Copyright (c) 2007 Thomas Porschberg <[email protected]> | Copyright (c) 2008- Patrick Collison <[email protected]> | Copyright 2007 by Nathan C. Myers <[email protected]>; some rights reserved | Copyright 2012 Twitter, Inc"
 core,github.com/aquasecurity/go-gem-version,Apache-2.0,Copyright (c) 2020 Teppei Fukuda (knqyf263)
@@ -1080,9 +1075,6 @@ core,github.com/goccy/go-json/internal/encoder/vm_color_indent,MIT,Copyright (c)
 core,github.com/goccy/go-json/internal/encoder/vm_indent,MIT,Copyright (c) 2020 Masaaki Goshima
 core,github.com/goccy/go-json/internal/errors,MIT,Copyright (c) 2020 Masaaki Goshima
 core,github.com/goccy/go-json/internal/runtime,MIT,Copyright (c) 2020 Masaaki Goshima
-core,github.com/gocolly/colly/v2,Apache-2.0,Copyright 2018 Adam Tauber
-core,github.com/gocolly/colly/v2/debug,Apache-2.0,Copyright 2018 Adam Tauber
-core,github.com/gocolly/colly/v2/storage,Apache-2.0,Copyright 2018 Adam Tauber
 core,github.com/gocomply/scap/pkg/scap/constants,CC0-1.0,CC0 1.0 Universal
 core,github.com/gocomply/scap/pkg/scap/models/cdf,CC0-1.0,CC0 1.0 Universal
 core,github.com/gocomply/scap/pkg/scap/models/cpe,CC0-1.0,CC0 1.0 Universal
@@ -1393,7 +1385,6 @@ core,github.com/justincormack/go-memfd/msyscall,MIT,Copyright (c) 2017 Justin Co
 core,github.com/kardianos/osext,BSD-3-Clause,Copyright (c) 2012 The Go Authors. All rights reserved
 core,github.com/karrick/godirwalk,BSD-2-Clause,"Copyright (c) 2017, Karrick McDermott"
 core,github.com/kballard/go-shellquote,MIT,Copyright (C) 2014 Kevin Ballard
-core,github.com/kennygrant/sanitize,BSD-3-Clause,Copyright (c) 2017 Mechanism Design. All rights reserved
 core,github.com/kevinburke/ssh_config,MIT,"Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton | Copyright (c) 2017 Kevin Burke"
 core,github.com/kjk/lzma,BSD-3-Clause,"Copyright (c) 2010, Andrei Vieru. All rights reserved"
 core,github.com/klauspost/compress,BSD-3-Clause,Copyright (c) 2011 The Snappy-Go Authors. All rights reserved | Copyright (c) 2012 The Go Authors. All rights reserved | Copyright (c) 2015 Klaus Post | Copyright (c) 2019 Klaus Post. All rights reserved | Copyright 2016 The filepathx Authors | Copyright 2016-2017 The New York Times Company
@@ -1988,7 +1979,6 @@ core,github.com/rs/zerolog/log,MIT,Copyright (c) 2017 Olivier Poitrey
 core,github.com/ryanuber/go-glob,MIT,Copyright (c) 2014 Ryan Uber
 core,github.com/sagikazarmark/locafero,MIT,Copyright (c) 2023 Márk Sági-Kazár <[email protected]>
 core,github.com/sagikazarmark/slog-shim,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
-core,github.com/saintfish/chardet,MIT,Copyright (c) 2012 chardet Authors | Sheng Yu (yusheng dot sjtu at gmail dot com)
 core,github.com/samber/lo,MIT,Copyright (c) 2022 Samuel Berthe | Copyright © 2022 [Samuel Berthe](https://github.com/samber)
 core,github.com/samber/lo/internal/constraints,MIT,Copyright (c) 2022 Samuel Berthe | Copyright © 2022 [Samuel Berthe](https://github.com/samber)
 core,github.com/samber/lo/internal/rand,MIT,Copyright (c) 2022 Samuel Berthe | Copyright © 2022 [Samuel Berthe](https://github.com/samber)
@@ -2127,7 +2117,6 @@ core,github.com/syndtr/goleveldb/leveldb/table,BSD-2-Clause,Copyright 2012 Surya
 core,github.com/syndtr/goleveldb/leveldb/util,BSD-2-Clause,Copyright 2012 Suryandaru Triandana <[email protected]>
 core,github.com/tchap/go-patricia/v2/patricia,MIT,Copyright (c) 2014 The AUTHORS | Ondřej Kupka <[email protected]> | This is the complete list of go-patricia copyright holders:
 core,github.com/tedsuo/rata,MIT,Copyright (c) 2014 Ted Young
-core,github.com/temoto/robotstxt,MIT,Copyright (c) 2010 Sergey Shepelev <[email protected]>
 core,github.com/tetratelabs/wazero,Apache-2.0,Copyright 2020-2023 wazero authors
 core,github.com/tetratelabs/wazero/api,Apache-2.0,Copyright 2020-2023 wazero authors
 core,github.com/tetratelabs/wazero/experimental,Apache-2.0,Copyright 2020-2023 wazero authors
@@ -2746,13 +2735,6 @@ core,google.golang.org/api/transport,BSD-3-Clause,Copyright (c) 2011 Google Inc.
 core,google.golang.org/api/transport/grpc,BSD-3-Clause,Copyright (c) 2011 Google Inc. All rights reserved.
 core,google.golang.org/api/transport/http,BSD-3-Clause,Copyright (c) 2011 Google Inc. All rights reserved.
 core,google.golang.org/api/transport/http/internal/propagation,BSD-3-Clause,Copyright (c) 2011 Google Inc. All rights reserved.
-core,google.golang.org/appengine/internal,Apache-2.0,Copyright 2011 Google Inc. All rights reserved.
-core,google.golang.org/appengine/internal/base,Apache-2.0,Copyright 2011 Google Inc. All rights reserved.
-core,google.golang.org/appengine/internal/datastore,Apache-2.0,Copyright 2011 Google Inc. All rights reserved.
-core,google.golang.org/appengine/internal/log,Apache-2.0,Copyright 2011 Google Inc. All rights reserved.
-core,google.golang.org/appengine/internal/remote_api,Apache-2.0,Copyright 2011 Google Inc. All rights reserved.
-core,google.golang.org/appengine/internal/urlfetch,Apache-2.0,Copyright 2011 Google Inc. All rights reserved.
-core,google.golang.org/appengine/urlfetch,Apache-2.0,Copyright 2011 Google Inc. All rights reserved.
 core,google.golang.org/genproto/googleapis/api,Apache-2.0,Copyright 2015 Google LLC
 core,google.golang.org/genproto/googleapis/api/annotations,Apache-2.0,Copyright 2015 Google LLC
 core,google.golang.org/genproto/googleapis/api/expr/v1alpha1,Apache-2.0,Copyright 2015 Google LLC

comp/otelcol/collector/impl-pipeline/flare_filler.go

@@ -20,8 +20,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gocolly/colly/v2"
-
 	flaretypes "github.com/DataDog/datadog-agent/comp/core/flare/types"
 	extension "github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/def"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
@@ -106,48 +104,11 @@ func (c *collectorImpl) fillFlare(fb flaretypes.FlareBuilder) error {
 				fb.AddFile(fmt.Sprintf("otel/otel-flare/%s.dat", name), data)
 			}
 
-			if !src.Crawl {
-				continue
-			}
-
-			// crawl the url by following any hyperlinks
-			col := colly.NewCollector()
-			col.OnHTML("a", func(e *colly.HTMLElement) {
-				// visit all links
-				link := e.Attr("href")
-				if err := e.Request.Visit(e.Request.AbsoluteURL(link)); err != nil {
-					filename := strings.ReplaceAll(url.PathEscape(link), ":", "_")
-					fb.AddFile(fmt.Sprintf("otel/otel-flare/crawl-%s.err", filename), []byte(err.Error()))
-				}
-			})
-			col.OnResponse(func(r *colly.Response) {
-				// the root sources (from the extension.Response) were already fetched earlier
-				// don't re-fetch them
-				responseURL := r.Request.URL.String()
-				if contains(sourceURLs, responseURL) {
-					return
-				}
-				// use the url as the basis for the filename saved in the flare
-				filename := strings.ReplaceAll(url.PathEscape(responseURL), ":", "_")
-				fb.AddFile(fmt.Sprintf("otel/otel-flare/crawl-%s", filename), r.Body)
-			})
-			if err := col.Visit(sourceURL); err != nil {
-				fb.AddFile("otel/otel-flare/crawl.err", []byte(err.Error()))
-			}
 		}
 	}
 	return nil
 }
 
-func contains(s []string, e string) bool {
-	for _, a := range s {
-		if a == e {
-			return true
-		}
-	}
-	return false
-}
-
 func toJSON(it interface{}) string {
 	data, err := json.Marshal(it)
 	if err != nil {

comp/otelcol/collector/impl-pipeline/flare_filler_test.go

@@ -11,7 +11,6 @@ package collectorimpl
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -41,8 +40,7 @@ func createFakeOTelExtensionHTTPServer() (string, func()) {
 			io.WriteString(w, "data-source-2")
 			return
 		} else if r.URL.Path == "/three" {
-			pageTmpl := `<body>Another source is <a href="%s/secret">here</a></body>`
-			io.WriteString(w, fmt.Sprintf(pageTmpl, testServerURL))
+			io.WriteString(w, "data-source-3")
 			return
 		} else if r.URL.Path == "/four" {
 			io.WriteString(w, "data-source-4")
@@ -93,27 +91,23 @@ func TestOTelExtFlareBuilder(t *testing.T) {
 		"prometheus": {
 			"url": [
 				"{{.url}}/one"
-			],
-			"crawl": false
+			]
 		},
 		"health_check": {
 			"url": [
 				"{{.url}}/two"
-			],
-			"crawl": false
+			]
 		},
 		"zpages": {
 			"url": [
 			"{{.url}}/three"
-			],
-			"crawl": true
+			]
 		},
 		"pprof": {
 			"url": [
 				"{{.url}}/four",
 				"{{.url}}/five/six"
-			],
-			"crawl": false
+			]
 		}
 	},
 	"environment": {{.environment}}
@@ -150,12 +144,9 @@ func TestOTelExtFlareBuilder(t *testing.T) {
 
 	f.AssertFileExists("otel", "otel-response.json")
 
-	// Template for the crawable page
-	pageTmpl := `<body>Another source is <a href="%s/secret">here</a></body>`
-
 	f.AssertFileContent("data-source-1", "otel/otel-flare/prometheus_one.dat")
 	f.AssertFileContent("data-source-2", "otel/otel-flare/health_check_two.dat")
-	f.AssertFileContent(fmt.Sprintf(pageTmpl, localServerURL), "otel/otel-flare/zpages_three.dat")
+	f.AssertFileContent("data-source-3", "otel/otel-flare/zpages_three.dat")
 	f.AssertFileContent("data-source-4", "otel/otel-flare/pprof_four.dat")
 	f.AssertFileContent("data-source-5-6", "otel/otel-flare/pprof_five_six.dat")
 

comp/otelcol/ddflareextension/def/types.go

@@ -24,8 +24,7 @@ type ConfigResponse struct {
 
 // OTelFlareSource is the response struct for flare debug sources
 type OTelFlareSource struct {
-	URLs  []string `json:"url"`
-	Crawl bool     `json:"crawl"`
+	URLs []string `json:"url"`
 }
 
 // DebugSourceResponse is the response struct for a map of OTelFlareSource

comp/otelcol/ddflareextension/impl/config.go

@@ -16,15 +16,14 @@ import (
 	"go.opentelemetry.io/collector/confmap"
 )
 
-type extractDebugEndpoint func(conf *confmap.Conf) (string, bool, error)
+type extractDebugEndpoint func(conf *confmap.Conf) (string, error)
 
 var (
 	errHTTPEndpointRequired  = errors.New("http endpoint required")
 	supportedDebugExtensions = map[string]extractDebugEndpoint{
 		"health_check": healthExtractEndpoint,
-		// disabled zpages from flare until solution to display data.
-		// "zpages":       zPagesExtractEndpoint,
-		"pprof": pprofExtractEndpoint,
+		"zpages":       zPagesExtractEndpoint,
+		"pprof":        pprofExtractEndpoint,
 	}
 )
 
@@ -58,20 +57,19 @@ func (c *Config) Unmarshal(conf *confmap.Conf) error {
 	return nil
 }
 
-// todo: uncomment once zpages data is re-added to flare
-// func zPagesExtractEndpoint(c *confmap.Conf) (string, bool, error) {
-// 	endpoint, err := regularStringEndpointExtractor(c)
-// 	return endpoint, true, err
-// }
+func zPagesExtractEndpoint(c *confmap.Conf) (string, error) {
+	endpoint, err := regularStringEndpointExtractor(c)
+	return endpoint, err
+}
 
-func pprofExtractEndpoint(c *confmap.Conf) (string, bool, error) {
+func pprofExtractEndpoint(c *confmap.Conf) (string, error) {
 	endpoint, err := regularStringEndpointExtractor(c)
-	return endpoint, false, err
+	return endpoint, err
 }
 
-func healthExtractEndpoint(c *confmap.Conf) (string, bool, error) {
+func healthExtractEndpoint(c *confmap.Conf) (string, error) {
 	endpoint, err := regularStringEndpointExtractor(c)
-	return endpoint, false, err
+	return endpoint, err
 }
 
 func regularStringEndpointExtractor(c *confmap.Conf) (string, error) {

comp/otelcol/ddflareextension/impl/config_test.go

@@ -73,21 +73,15 @@ func TestExtractors(t *testing.T) {
 
 	myConfMap := confmap.NewFromStringMap(m)
 
-	for extension, extractor := range supportedDebugExtensions {
-		expectedCrawl := false
-		if extension == "zpages" {
-			expectedCrawl = true
-		}
-
-		uri, crawl, err := extractor(myConfMap)
+	for _, extractor := range supportedDebugExtensions {
+		uri, err := extractor(myConfMap)
 		assert.NoError(t, err)
-		assert.Equal(t, expectedCrawl, crawl)
 		assert.Equal(t, endpoint, uri)
 	}
 
 	myConfMap = confmap.New()
 	for _, extractor := range supportedDebugExtensions {
-		_, _, err := extractor(myConfMap)
+		_, err := extractor(myConfMap)
 		assert.Error(t, err)
 	}
 

comp/otelcol/ddflareextension/impl/extension.go

@@ -87,23 +87,36 @@ func (ext *ddExtension) Start(_ context.Context, host component.Host) error {
 			continue
 		}
 
-		uri, crawl, err := extractor(exconf)
+		uri, err := extractor(exconf)
 
 		var uris []string
-		if extension.Type().String() == "pprof" {
-			uris = []string{uri + "/debug/pprof/heap", uri + "/debug/pprof/allocs", uri + "/debug/pprof/profile"}
-		} else {
+		switch extension.Type().String() {
+		case "pprof":
+			uris = []string{
+				uri + "/debug/pprof/heap",
+				uri + "/debug/pprof/allocs",
+				uri + "/debug/pprof/profile",
+			}
+		case "zpages":
+			uris = []string{
+				uri + "/debug/servicez",
+				uri + "/debug/pipelinez",
+				uri + "/debug/extensionz",
+				uri + "/debug/featurez",
+				uri + "/debug/tracez",
+			}
+		default:
 			uris = []string{uri}
 		}
 
 		if err != nil {
 			ext.telemetry.Logger.Info("Unavailable debug extension for", zap.String("extension", extension.String()))
-		} else {
-			ext.telemetry.Logger.Info("Found debug extension at", zap.String("uri", uri))
-			ext.debug.Sources[extension.String()] = extensionDef.OTelFlareSource{
-				URLs:  uris,
-				Crawl: crawl,
-			}
+			continue
+		}
+
+		ext.telemetry.Logger.Info("Found debug extension at", zap.String("uri", uri))
+		ext.debug.Sources[extension.String()] = extensionDef.OTelFlareSource{
+			URLs: uris,
 		}
 	}
 

go.mod

@@ -708,7 +708,6 @@ require (
 	github.com/elastic/go-seccomp-bpf v1.4.0
 	github.com/fatih/structtag v1.2.0
 	github.com/glaslos/ssdeep v0.4.0
-	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gocomply/scap v0.1.2-0.20230531064509-55a00f73e8d6
 	github.com/godror/godror v0.37.0
 	github.com/jackc/pgx/v5 v5.6.0
@@ -771,16 +770,11 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.0 // indirect
 	github.com/Intevation/gval v1.3.0 // indirect
 	github.com/Intevation/jsonpath v0.2.1 // indirect
-	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/Showmax/go-fqdn v1.0.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
-	github.com/andybalholm/cascadia v1.3.2 // indirect
-	github.com/antchfx/htmlquery v1.3.0 // indirect
-	github.com/antchfx/xmlquery v1.3.1 // indirect
-	github.com/antchfx/xpath v1.2.3 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/apache/thrift v0.20.0 // indirect
 	github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48 // indirect
@@ -854,7 +848,6 @@ require (
 	github.com/jaegertracing/jaeger v1.58.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/knadh/koanf/v2 v2.1.1 // indirect
 	github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect
@@ -935,7 +928,6 @@ require (
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 // indirect
 	github.com/shirou/gopsutil/v4 v4.24.5 // indirect
@@ -951,7 +943,6 @@ require (
 	github.com/stormcat24/protodep v0.1.8 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/swaggest/refl v1.3.0 // indirect
-	github.com/temoto/robotstxt v1.1.1 // indirect
 	github.com/tetratelabs/wazero v1.7.0 // indirect
 	github.com/tidwall/gjson v1.17.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect