[Filebeat] Fix hardcoded amazonaws.com endpoint (elastic#24861) (elas…

…tic#25010) (cherry picked from commit 7129c21) Co-authored-by: Alex Resnick <[email protected]>
leweafan · Apr 10, 2021 · cb71daf · cb71daf
1 parent d481339
commit cb71daf
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 10 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -206,6 +206,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
 - Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
 - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799]
+- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/input/awss3/collector.go b/x-pack/filebeat/input/awss3/collector.go
@@ -226,14 +226,20 @@ func (c *s3Collector) changeVisibilityTimeout(queueURL string, visibilityTimeout
 	return err
 }
 
-func getRegionFromQueueURL(queueURL string) (string, error) {
+func getRegionFromQueueURL(queueURL string, endpoint string) (string, error) {
 	// get region from queueURL
 	// Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs
-	queueURLSplit := strings.Split(queueURL, ".")
-	if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" {
-		return queueURLSplit[1], nil
+	url, err := url.Parse(queueURL)
+	if err != nil {
+		return "", fmt.Errorf(queueURL + " is not a valid URL")
+	}
+	if url.Scheme == "https" && url.Host != "" {
+		queueHostSplit := strings.Split(url.Host, ".")
+		if len(queueHostSplit) > 2 && (strings.Join(queueHostSplit[2:], ".") == endpoint || (endpoint == "" && queueHostSplit[2] == "amazonaws")) {
+			return queueHostSplit[1], nil
+		}
 	}
-	return "", fmt.Errorf("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}")
+	return "", fmt.Errorf("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}")
 }
 
 // handle message

diff --git a/x-pack/filebeat/input/awss3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go
@@ -58,10 +58,68 @@ func (m *MockS3Client) GetObjectRequest(input *s3.GetObjectInput) s3.GetObjectRe
 }
 
 func TestGetRegionFromQueueURL(t *testing.T) {
-	queueURL := "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs"
-	regionName, err := getRegionFromQueueURL(queueURL)
-	assert.NoError(t, err)
-	assert.Equal(t, "us-east-1", regionName)
+	casesPositive := []struct {
+		title          string
+		queueURL       string
+		endpoint       string
+		expectedRegion string
+	}{
+		{
+			"QueueURL using amazonaws.com domain with blank Endpoint",
+			"https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs",
+			"",
+			"us-east-1",
+		},
+		{
+			"QueueURL using abc.xyz and domain with matching Endpoint",
+			"https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs",
+			"abc.xyz",
+			"us-east-1",
+		},
+	}
+
+	for _, c := range casesPositive {
+		t.Run(c.title, func(t *testing.T) {
+			regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint)
+			assert.NoError(t, err)
+			assert.Equal(t, c.expectedRegion, regionName)
+		})
+	}
+
+	casesNegative := []struct {
+		title          string
+		queueURL       string
+		endpoint       string
+		expectedRegion string
+	}{
+		{
+			"QueueURL using abc.xyz and domain with blank Endpoint",
+			"https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs",
+			"",
+			"",
+		},
+		{
+			"QueueURL using abc.xyz and domain with different Endpoint",
+			"https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs",
+			"googlecloud.com",
+			"",
+		},
+		{
+			"QueueURL is an invalid URL",
+			":foo",
+			"",
+			"",
+		},
+	}
+
+	for _, c := range casesNegative {
+		t.Run(c.title, func(t *testing.T) {
+			regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint)
+			assert.Error(t, err)
+			assert.Empty(t, regionName)
+		})
+	}
+
 }
 
 func TestHandleMessage(t *testing.T) {

diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go
@@ -87,7 +87,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C
 		return nil, err
 	}
 
-	regionName, err := getRegionFromQueueURL(in.config.QueueURL)
+	regionName, err := getRegionFromQueueURL(in.config.QueueURL, in.config.AwsConfig.Endpoint)
 	if err != nil {
 		err := fmt.Errorf("getRegionFromQueueURL failed: %w", err)
 		log.Error(err)