Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improved ECS attribute and origin translation in awsxrayexporter #1428

Merged
merged 6 commits into from
Nov 5, 2020
Merged

Improved ECS attribute and origin translation in awsxrayexporter #1428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dc73a66
initial commit of ecs translation
Oct 29, 2020
d2bfc7e
reverted trace ID change
Oct 29, 2020
ed27d55
added unit tests
Oct 30, 2020
f03b0df
addressed comments and unit test issues
Nov 3, 2020
8923b17
Merge branch 'master' into xray-translate
willarmiros Nov 4, 2020
c31c6bf
fixed breaking API changes
Nov 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 89 additions & 7 deletions exporter/awsxrayexporter/translator/aws.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package translator

import (
"bytes"
"strconv"

"github.com/aws/aws-sdk-go/aws"
Expand All @@ -24,9 +25,21 @@ import (
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/awsxray"
)

const (
attributeInfrastructureService = "cloud.infrastructure_service"
awsEcsClusterArn = "aws.ecs.cluster.arn"
awsEcsContainerArn = "aws.ecs.container.arn"
awsEcsTaskArn = "aws.ecs.task.arn"
awsEcsTaskFamily = "aws.ecs.task.family"
awsEcsLaunchType = "aws.ecs.launchtype"
awsLogGroupNames = "aws.log.group.names"
awsLogGroupArns = "aws.log.group.arns"
)

func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]string, *awsxray.AWSData) {
var (
cloud string
service string
account string
zone string
hostID string
Expand All @@ -49,6 +62,14 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
containerID string
clusterName string
podUID string
clusterArn string
containerArn string
taskArn string
taskFamily string
launchType string
logGroups pdata.AnyValueArray
logGroupArns pdata.AnyValueArray
cwl []awsxray.LogGroupMetadata
ec2 *awsxray.EC2Metadata
ecs *awsxray.ECSMetadata
ebs *awsxray.BeanstalkMetadata
Expand All @@ -61,6 +82,8 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
switch key {
case semconventions.AttributeCloudProvider:
cloud = value.StringVal()
case attributeInfrastructureService:
service = value.StringVal()
case semconventions.AttributeCloudAccount:
account = value.StringVal()
case semconventions.AttributeCloudZone:
Expand Down Expand Up @@ -95,6 +118,20 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
containerID = value.StringVal()
case semconventions.AttributeK8sCluster:
clusterName = value.StringVal()
case awsEcsClusterArn:
clusterArn = value.StringVal()
case awsEcsContainerArn:
containerArn = value.StringVal()
case awsEcsTaskArn:
taskArn = value.StringVal()
case awsEcsTaskFamily:
taskFamily = value.StringVal()
case awsEcsLaunchType:
launchType = value.StringVal()
case awsLogGroupNames:
logGroups = value.ArrayVal()
case awsLogGroupArns:
logGroupArns = value.ArrayVal()
}
})
}
Expand Down Expand Up @@ -125,25 +162,32 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
filtered[key] = value
}
}
if cloud != "aws" && cloud != "" {
if cloud != semconventions.AttributeCloudProviderAWS && cloud != "" {
return filtered, nil // not AWS so return nil
}
// progress from least specific to most specific origin so most specific ends up as origin
// as per X-Ray docs
if hostID != "" {

if service == "EC2" || hostID != "" {
ec2 = &awsxray.EC2Metadata{
InstanceID: awsxray.String(hostID),
AvailabilityZone: awsxray.String(zone),
InstanceSize: awsxray.String(hostType),
AmiID: awsxray.String(amiID),
}
}
if container != "" {
if service == "ECS" || container != "" {
ecs = &awsxray.ECSMetadata{
ContainerName: awsxray.String(container),
ContainerID: awsxray.String(containerID),
ContainerName: awsxray.String(container),
ContainerID: awsxray.String(containerID),
AvailabilityZone: awsxray.String(zone),
ContainerArn: awsxray.String(containerArn),
ClusterArn: awsxray.String(clusterArn),
TaskArn: awsxray.String(taskArn),
TaskFamily: awsxray.String(taskFamily),
LaunchType: awsxray.String(launchType),
}
}

// TODO(willarmiros): Add infrastructure_service checks once their resource detectors are implemented
if deployID != "" {
deployNum, err := strconv.ParseInt(deployID, 10, 64)
if err != nil {
Expand All @@ -163,6 +207,14 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
}
}

// Since we must couple log group ARNs and Log Group Names in the same CWLogs object, we first try to derive the
// names from the ARN, then fall back to just recording the names
if logGroupArns != (pdata.AnyValueArray{}) && logGroupArns.Len() > 0 {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need this != even though we have a len check?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I initially didn't have it and it turns out the Len() method relies on an underlying pointer orig in the AnyValueArray struct, so if we don't initialize logGroupArns we get a nil pointer exception. I could check for nilness of orig, but it seemed like an implementation detail that I shouldn't be referencing directly.

cwl = getLogGroupMetadata(logGroupArns, true)
} else if logGroups != (pdata.AnyValueArray{}) && logGroups.Len() > 0 {
cwl = getLogGroupMetadata(logGroups, false)
}

if sdkName != "" && sdkLanguage != "" {
// Convention for SDK name for xray SDK information is e.g., `X-Ray SDK for Java`, `X-Ray for Go`.
// We fill in with e.g, `opentelemetry for java` by using the conventions
Expand All @@ -180,6 +232,7 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
awsData := &awsxray.AWSData{
AccountID: awsxray.String(account),
Beanstalk: ebs,
CWLogs: cwl,
ECS: ecs,
EC2: ec2,
EKS: eks,
Expand All @@ -192,3 +245,32 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
}
return filtered, awsData
}

// Given an array of log group ARNs, create a corresponding amount of LogGroupMetadata objects with log_group and arn
// populated, or given an array of just log group names, create the LogGroupMetadata objects with arn omitted
func getLogGroupMetadata(logGroups pdata.AnyValueArray, isArn bool) []awsxray.LogGroupMetadata {
var lgm []awsxray.LogGroupMetadata
for i := 0; i < logGroups.Len(); i++ {
if isArn {
lgm = append(lgm, awsxray.LogGroupMetadata{
Arn: awsxray.String(logGroups.At(i).StringVal()),
LogGroup: awsxray.String(parseLogGroup(logGroups.At(i).StringVal())),
})
} else {
lgm = append(lgm, awsxray.LogGroupMetadata{
LogGroup: awsxray.String(logGroups.At(i).StringVal()),
})
}
}

return lgm
}

func parseLogGroup(arn string) string {
i := bytes.LastIndexByte([]byte(arn), byte(':'))
if i != -1 {
return arn[i+1:]
}

return arn
}
84 changes: 81 additions & 3 deletions exporter/awsxrayexporter/translator/aws_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func TestAwsFromEc2Resource(t *testing.T) {
resource.InitEmpty()
attrs := pdata.NewAttributeMap()
attrs.InsertString(semconventions.AttributeCloudProvider, semconventions.AttributeCloudProviderAWS)
attrs.InsertString(attributeInfrastructureService, "EC2")
attrs.InsertString(semconventions.AttributeCloudAccount, "123456789")
attrs.InsertString(semconventions.AttributeCloudZone, "us-east-1c")
attrs.InsertString(semconventions.AttributeHostID, instanceID)
Expand Down Expand Up @@ -63,12 +64,19 @@ func TestAwsFromEcsResource(t *testing.T) {
instanceID := "i-00f7c0bcb26da2a99"
containerName := "signup_aggregator-x82ufje83"
containerID := "0123456789A"
az := "us-east-1c"
launchType := "fargate"
family := "family"
taskArn := "arn:aws:ecs:us-west-2:123456789123:task/123"
clusterArn := "arn:aws:ecs:us-west-2:123456789123:cluster/my-cluster"
containerArn := "arn:aws:ecs:us-west-2:123456789123:container-instance/123"
resource := pdata.NewResource()
resource.InitEmpty()
attrs := pdata.NewAttributeMap()
attrs.InsertString(semconventions.AttributeCloudProvider, semconventions.AttributeCloudProviderAWS)
attrs.InsertString(attributeInfrastructureService, "ECS")
attrs.InsertString(semconventions.AttributeCloudAccount, "123456789")
attrs.InsertString(semconventions.AttributeCloudZone, "us-east-1c")
attrs.InsertString(semconventions.AttributeCloudZone, az)
attrs.InsertString(semconventions.AttributeContainerImage, "otel/signupaggregator")
attrs.InsertString(semconventions.AttributeContainerTag, "v1")
attrs.InsertString(semconventions.AttributeK8sCluster, "production")
Expand All @@ -78,7 +86,13 @@ func TestAwsFromEcsResource(t *testing.T) {
attrs.InsertString(semconventions.AttributeContainerName, containerName)
attrs.InsertString(semconventions.AttributeContainerID, containerID)
attrs.InsertString(semconventions.AttributeHostID, instanceID)
attrs.InsertString(awsEcsClusterArn, clusterArn)
attrs.InsertString(awsEcsContainerArn, containerArn)
attrs.InsertString(awsEcsTaskArn, taskArn)
attrs.InsertString(awsEcsTaskFamily, family)
attrs.InsertString(awsEcsLaunchType, launchType)
attrs.InsertString(semconventions.AttributeHostType, "m5.xlarge")

attrs.CopyTo(resource.Attributes())

attributes := make(map[string]string)
Expand All @@ -92,8 +106,14 @@ func TestAwsFromEcsResource(t *testing.T) {
assert.Nil(t, awsData.Beanstalk)
assert.NotNil(t, awsData.EKS)
assert.Equal(t, &awsxray.ECSMetadata{
ContainerName: aws.String(containerName),
ContainerID: aws.String(containerID),
ContainerName: aws.String(containerName),
ContainerID: aws.String(containerID),
AvailabilityZone: aws.String(az),
ClusterArn: aws.String(clusterArn),
ContainerArn: aws.String(containerArn),
TaskArn: aws.String(taskArn),
TaskFamily: aws.String(family),
LaunchType: aws.String(launchType),
}, awsData.ECS)
}

Expand Down Expand Up @@ -343,3 +363,61 @@ func TestCustomSDK(t *testing.T) {
assert.Equal(t, "opentracing for java", *awsData.XRay.SDK)
assert.Equal(t, "2.0.3", *awsData.XRay.SDKVersion)
}

func TestLogGroups(t *testing.T) {
cwl1 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group1"),
}
cwl2 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group2"),
}

attributes := make(map[string]string)
resource := pdata.NewResource()
resource.InitEmpty()
lg := pdata.NewAttributeValueArray()
ava := lg.ArrayVal()
ava.Append(pdata.NewAttributeValueString("group1"))
ava.Append(pdata.NewAttributeValueString("group2"))

resource.Attributes().Insert(awsLogGroupNames, lg)

filtered, awsData := makeAws(attributes, resource)

assert.NotNil(t, filtered)
assert.NotNil(t, awsData)
assert.Equal(t, 2, len(awsData.CWLogs))
assert.Contains(t, awsData.CWLogs, cwl1)
assert.Contains(t, awsData.CWLogs, cwl2)
}

func TestLogGroupsFromArns(t *testing.T) {
group1 := "arn:aws:logs:us-east-1:123456789123:log-group:group1"
cwl1 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group1"),
Arn: awsxray.String(group1),
}
group2 := "arn:aws:logs:us-east-1:123456789123:log-group:group2"
cwl2 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group2"),
Arn: awsxray.String(group2),
}

attributes := make(map[string]string)
resource := pdata.NewResource()
resource.InitEmpty()
lga := pdata.NewAttributeValueArray()
ava := lga.ArrayVal()
ava.Append(pdata.NewAttributeValueString(group1))
ava.Append(pdata.NewAttributeValueString(group2))

resource.Attributes().Insert(awsLogGroupArns, lga)

filtered, awsData := makeAws(attributes, resource)

assert.NotNil(t, filtered)
assert.NotNil(t, awsData)
assert.Equal(t, 2, len(awsData.CWLogs))
assert.Contains(t, awsData.CWLogs, cwl1)
assert.Contains(t, awsData.CWLogs, cwl2)
}
47 changes: 42 additions & 5 deletions exporter/awsxrayexporter/translator/segment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,12 @@ import (

// AWS X-Ray acceptable values for origin field.
const (
OriginEC2 = "AWS::EC2::Instance"
OriginECS = "AWS::ECS::Container"
OriginEB = "AWS::ElasticBeanstalk::Environment"
OriginEKS = "AWS::EKS::Container"
OriginEC2 = "AWS::EC2::Instance"
OriginECS = "AWS::ECS::Container"
OriginECSEC2 = "AWS::ECS::EC2"
OriginECSFargate = "AWS::ECS::Fargate"
OriginEB = "AWS::ElasticBeanstalk::Environment"
OriginEKS = "AWS::EKS::Container"
)

var (
Expand Down Expand Up @@ -230,6 +232,37 @@ func determineAwsOrigin(resource pdata.Resource) string {
return ""
}
}

// TODO(willarmiros): Only use infrastructure_service for origin resolution once detectors for all AWS environments are
// implemented for robustness
if is, present := resource.Attributes().Get("cloud.infrastructure_service"); present {
switch is.StringVal() {
case "EKS":
return OriginEKS
case "ElasticBeanstalk":
return OriginEB
case "ECS":
lt, present := resource.Attributes().Get("aws.ecs.launchtype")
if !present {
return OriginECS
}
switch lt.StringVal() {
case "ec2":
return OriginECSEC2
case "fargate":
return OriginECSFargate
default:
return OriginECS
}
case "EC2":
return OriginEC2

// If infrastructure_service is defined with a non-AWS value, we should not assign it an AWS origin
default:
return ""
}
}

// EKS > EB > ECS > EC2
_, eks := resource.Attributes().Get(semconventions.AttributeK8sCluster)
if eks {
Expand All @@ -243,7 +276,11 @@ func determineAwsOrigin(resource pdata.Resource) string {
if ecs {
return OriginECS
}
return OriginEC2
_, ec2 := resource.Attributes().Get(semconventions.AttributeHostID)
if ec2 {
return OriginEC2
}
return ""
}

// convertToAmazonTraceID converts a trace ID to the Amazon format.
Expand Down
Loading