From 47cf0c8b7d705a7ec3dacdebaf8ca962539e3c8d Mon Sep 17 00:00:00 2001 From: minherz Date: Fri, 13 May 2022 00:25:07 +0000 Subject: [PATCH] fix(logging): revise automatic resource detection for ingested logs (#6022) align resource detection with heuristics in logging libraries for other languages: - use /sys/class/dmi/id/product_name to read a product name of the resource on Linux - change resource detection heuristics in a way that the order of validation is not important - reduce timeout and retrying logic when querying metadata server - demand active metadata server for detecting GCP resources (for GCE, GAE, GKE, CR and CF) - add test to validate resource detection heuristics --- logging/internal/environment.go | 75 ++++++++++ logging/resource.go | 252 ++++++++++++++++--------------- logging/resource_test.go | 255 ++++++++++++++++++++++++++++++++ 3 files changed, 466 insertions(+), 116 deletions(-) create mode 100644 logging/internal/environment.go create mode 100644 logging/resource_test.go diff --git a/logging/internal/environment.go b/logging/internal/environment.go new file mode 100644 index 000000000000..309f9faa3a03 --- /dev/null +++ b/logging/internal/environment.go @@ -0,0 +1,75 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "io/ioutil" + "net" + "net/http" + "os" + "strings" + "time" + + "cloud.google.com/go/compute/metadata" +) + +// ResourceAtttributesGetter abstracts environment lookup methods to query for environment variables, metadata attributes and file content. +type ResourceAtttributesGetter interface { + EnvVar(name string) string + Metadata(path string) string + ReadAll(path string) string +} + +var getter ResourceAtttributesGetter = &defaultResourceGetter{ + metaClient: metadata.NewClient(&http.Client{ + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 1 * time.Second, + KeepAlive: 10 * time.Second, + }).Dial, + }, + })} + +// ResourceAttributes provides read-only access to the ResourceAtttributesGetter interface implementation. +func ResourceAttributes() ResourceAtttributesGetter { + return getter +} + +type defaultResourceGetter struct { + metaClient *metadata.Client +} + +// EnvVar uses os.LookupEnv() to lookup for environment variable by name. +func (g *defaultResourceGetter) EnvVar(name string) string { + return os.Getenv(name) +} + +// Metadata uses metadata package Client.Get() to lookup for metadata attributes by path. +func (g *defaultResourceGetter) Metadata(path string) string { + val, err := g.metaClient.Get(path) + if err != nil { + return "" + } + return strings.TrimSpace(val) +} + +// ReadAll reads all content of the file as a string. +func (g *defaultResourceGetter) ReadAll(path string) string { + bytes, err := ioutil.ReadFile(path) + if err != nil { + return "" + } + return string(bytes) +} diff --git a/logging/resource.go b/logging/resource.go index b4e35fa7c4b1..6040a960998d 100644 --- a/logging/resource.go +++ b/logging/resource.go @@ -15,12 +15,11 @@ package logging import ( - "io/ioutil" - "os" + "runtime" "strings" "sync" - "cloud.google.com/go/compute/metadata" + "cloud.google.com/go/logging/internal" mrpb "google.golang.org/genproto/googleapis/api/monitoredres" ) @@ -34,165 +33,180 @@ type commonResource struct{ *mrpb.MonitoredResource } func (r commonResource) set(l *Logger) { l.commonResource = r.MonitoredResource } -var detectedResource struct { - pb *mrpb.MonitoredResource - once sync.Once +type resource struct { + pb *mrpb.MonitoredResource + attrs internal.ResourceAtttributesGetter + once *sync.Once } -// isAppEngine returns true for both standard and flex -func isAppEngine() bool { - _, service := os.LookupEnv("GAE_SERVICE") - _, version := os.LookupEnv("GAE_VERSION") - _, instance := os.LookupEnv("GAE_INSTANCE") +var detectedResource = &resource{ + attrs: internal.ResourceAttributes(), + once: new(sync.Once), +} - return service && version && instance +func (r *resource) metadataProjectID() string { + return r.attrs.Metadata("project/project-id") } -func detectAppEngineResource() *mrpb.MonitoredResource { - projectID, err := metadata.ProjectID() - if err != nil { - return nil +func (r *resource) metadataZone() string { + zone := r.attrs.Metadata("instance/zone") + if zone != "" { + return zone[strings.LastIndex(zone, "/")+1:] + } + return "" +} + +func (r *resource) metadataRegion() string { + region := r.attrs.Metadata("instance/region") + if region != "" { + return region[strings.LastIndex(region, "/")+1:] } + return "" +} + +// isMetadataActive queries valid response on "/computeMetadata/v1/" URL +func (r *resource) isMetadataActive() bool { + data := r.attrs.Metadata("") + return data != "" +} + +// isAppEngine returns true for both standard and flex +func (r *resource) isAppEngine() bool { + service := r.attrs.EnvVar("GAE_SERVICE") + version := r.attrs.EnvVar("GAE_VERSION") + instance := r.attrs.EnvVar("GAE_INSTANCE") + return service != "" && version != "" && instance != "" +} + +func detectAppEngineResource() *mrpb.MonitoredResource { + projectID := detectedResource.metadataProjectID() if projectID == "" { - projectID = os.Getenv("GOOGLE_CLOUD_PROJECT") + projectID = detectedResource.attrs.EnvVar("GOOGLE_CLOUD_PROJECT") } - zone, err := metadata.Zone() - if err != nil { + if projectID == "" { return nil } + zone := detectedResource.metadataZone() + service := detectedResource.attrs.EnvVar("GAE_SERVICE") + version := detectedResource.attrs.EnvVar("GAE_VERSION") return &mrpb.MonitoredResource{ Type: "gae_app", Labels: map[string]string{ - "project_id": projectID, - "module_id": os.Getenv("GAE_SERVICE"), - "version_id": os.Getenv("GAE_VERSION"), - "instance_id": os.Getenv("GAE_INSTANCE"), - "runtime": os.Getenv("GAE_RUNTIME"), - "zone": zone, + "project_id": projectID, + "module_id": service, + "version_id": version, + "zone": zone, }, } } -func isCloudFunction() bool { - // Reserved envvars in older function runtimes, e.g. Node.js 8, Python 3.7 and Go 1.11. - _, name := os.LookupEnv("FUNCTION_NAME") - _, region := os.LookupEnv("FUNCTION_REGION") - _, entry := os.LookupEnv("ENTRY_POINT") - - // Reserved envvars in newer function runtimes. - _, target := os.LookupEnv("FUNCTION_TARGET") - _, signature := os.LookupEnv("FUNCTION_SIGNATURE_TYPE") - _, service := os.LookupEnv("K_SERVICE") - return (name && region && entry) || (target && signature && service) +func (r *resource) isCloudFunction() bool { + target := r.attrs.EnvVar("FUNCTION_TARGET") + signature := r.attrs.EnvVar("FUNCTION_SIGNATURE_TYPE") + // note that this envvar is also present in Cloud Run environments + service := r.attrs.EnvVar("K_SERVICE") + return target != "" && signature != "" && service != "" } func detectCloudFunction() *mrpb.MonitoredResource { - projectID, err := metadata.ProjectID() - if err != nil { - return nil - } - zone, err := metadata.Zone() - if err != nil { + projectID := detectedResource.metadataProjectID() + if projectID == "" { return nil } - // Newer functions runtimes store name in K_SERVICE. - functionName, exists := os.LookupEnv("K_SERVICE") - if !exists { - functionName, _ = os.LookupEnv("FUNCTION_NAME") - } + region := detectedResource.metadataRegion() + functionName := detectedResource.attrs.EnvVar("K_SERVICE") return &mrpb.MonitoredResource{ Type: "cloud_function", Labels: map[string]string{ "project_id": projectID, - "region": regionFromZone(zone), + "region": region, "function_name": functionName, }, } } -func isCloudRun() bool { - _, config := os.LookupEnv("K_CONFIGURATION") - _, service := os.LookupEnv("K_SERVICE") - _, revision := os.LookupEnv("K_REVISION") - return config && service && revision +func (r *resource) isCloudRun() bool { + config := r.attrs.EnvVar("K_CONFIGURATION") + // note that this envvar is also present in Cloud Function environments + service := r.attrs.EnvVar("K_SERVICE") + revision := r.attrs.EnvVar("K_REVISION") + return config != "" && service != "" && revision != "" } func detectCloudRunResource() *mrpb.MonitoredResource { - projectID, err := metadata.ProjectID() - if err != nil { - return nil - } - zone, err := metadata.Zone() - if err != nil { + projectID := detectedResource.metadataProjectID() + if projectID == "" { return nil } + region := detectedResource.metadataRegion() + config := detectedResource.attrs.EnvVar("K_CONFIGURATION") + service := detectedResource.attrs.EnvVar("K_SERVICE") + revision := detectedResource.attrs.EnvVar("K_REVISION") return &mrpb.MonitoredResource{ Type: "cloud_run_revision", Labels: map[string]string{ "project_id": projectID, - "location": regionFromZone(zone), - "service_name": os.Getenv("K_SERVICE"), - "revision_name": os.Getenv("K_REVISION"), - "configuration_name": os.Getenv("K_CONFIGURATION"), + "location": region, + "service_name": service, + "revision_name": revision, + "configuration_name": config, }, } } -func isKubernetesEngine() bool { - clusterName, err := metadata.InstanceAttributeValue("cluster-name") - // Note: InstanceAttributeValue can return "", nil - if err != nil || clusterName == "" { +func (r *resource) isKubernetesEngine() bool { + clusterName := r.attrs.Metadata("instance/attributes/cluster-name") + if clusterName == "" { return false } return true } func detectKubernetesResource() *mrpb.MonitoredResource { - projectID, err := metadata.ProjectID() - if err != nil { - return nil - } - zone, err := metadata.Zone() - if err != nil { - return nil - } - clusterName, err := metadata.InstanceAttributeValue("cluster-name") - if err != nil { + projectID := detectedResource.metadataProjectID() + if projectID == "" { return nil } - namespaceBytes, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") - namespaceName := "" - if err == nil { - namespaceName = string(namespaceBytes) + zone := detectedResource.metadataZone() + clusterName := detectedResource.attrs.Metadata("instance/attributes/cluster-name") + namespaceName := detectedResource.attrs.ReadAll("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if namespaceName == "" { + // if automountServiceAccountToken is disabled allow to customize + // the namespace via environment + namespaceName = detectedResource.attrs.EnvVar("NAMESPACE_NAME") } + // note: if deployment customizes hostname, HOSTNAME envvar will have invalid content + podName := detectedResource.attrs.EnvVar("HOSTNAME") + // there is no way to derive container name from within container; use custom envvar if available + containerName := detectedResource.attrs.EnvVar("CONTAINER_NAME") return &mrpb.MonitoredResource{ Type: "k8s_container", Labels: map[string]string{ "cluster_name": clusterName, "location": zone, "project_id": projectID, - "pod_name": os.Getenv("HOSTNAME"), + "pod_name": podName, "namespace_name": namespaceName, - // To get the `container_name` label, users need to explicitly provide it. - "container_name": os.Getenv("CONTAINER_NAME"), + "container_name": containerName, }, } } -func detectGCEResource() *mrpb.MonitoredResource { - projectID, err := metadata.ProjectID() - if err != nil { - return nil - } - id, err := metadata.InstanceID() - if err != nil { - return nil - } - zone, err := metadata.Zone() - if err != nil { +func (r *resource) isComputeEngine() bool { + preempted := r.attrs.Metadata("instance/preempted") + platform := r.attrs.Metadata("instance/cpu-platform") + appBucket := r.attrs.Metadata("instance/attributes/gae_app_bucket") + return preempted != "" && platform != "" && appBucket == "" +} + +func detectComputeEngineResource() *mrpb.MonitoredResource { + projectID := detectedResource.metadataProjectID() + if projectID == "" { return nil } + id := detectedResource.attrs.Metadata("instance/id") + zone := detectedResource.metadataZone() return &mrpb.MonitoredResource{ Type: "gce_instance", Labels: map[string]string{ @@ -205,24 +219,38 @@ func detectGCEResource() *mrpb.MonitoredResource { func detectResource() *mrpb.MonitoredResource { detectedResource.once.Do(func() { - switch { - // AppEngine, Functions, CloudRun, Kubernetes are detected first, - // as metadata.OnGCE() erroneously returns true on these runtimes. - case isAppEngine(): - detectedResource.pb = detectAppEngineResource() - case isCloudFunction(): - detectedResource.pb = detectCloudFunction() - case isCloudRun(): - detectedResource.pb = detectCloudRunResource() - case isKubernetesEngine(): - detectedResource.pb = detectKubernetesResource() - case metadata.OnGCE(): - detectedResource.pb = detectGCEResource() + if detectedResource.isMetadataActive() { + name := systemProductName() + switch { + case name == "Google App Engine", detectedResource.isAppEngine(): + detectedResource.pb = detectAppEngineResource() + case name == "Google Cloud Functions", detectedResource.isCloudFunction(): + detectedResource.pb = detectCloudFunction() + case name == "Google Cloud Run", detectedResource.isCloudRun(): + detectedResource.pb = detectCloudRunResource() + // cannot use name validation for GKE and GCE because + // both of them set product name to "Google Compute Engine" + case detectedResource.isKubernetesEngine(): + detectedResource.pb = detectKubernetesResource() + case detectedResource.isComputeEngine(): + detectedResource.pb = detectComputeEngineResource() + } } }) return detectedResource.pb } +// systemProductName reads resource type on the Linux-based environments such as +// Cloud Functions, Cloud Run, GKE, GCE, GAE, etc. +func systemProductName() string { + if runtime.GOOS != "linux" { + // We don't have any non-Linux clues available, at least yet. + return "" + } + slurp := detectedResource.attrs.ReadAll("/sys/class/dmi/id/product_name") + return strings.TrimSpace(slurp) +} + var resourceInfo = map[string]struct{ rtype, label string }{ "organizations": {"organization", "organization_id"}, "folders": {"folder", "folder_id"}, @@ -245,14 +273,6 @@ func monitoredResource(parent string) *mrpb.MonitoredResource { } } -func regionFromZone(zone string) string { - cutoff := strings.LastIndex(zone, "-") - if cutoff > 0 { - return zone[:cutoff] - } - return zone -} - func globalResource(projectID string) *mrpb.MonitoredResource { return &mrpb.MonitoredResource{ Type: "global", diff --git a/logging/resource_test.go b/logging/resource_test.go new file mode 100644 index 000000000000..edd04cfaa3b0 --- /dev/null +++ b/logging/resource_test.go @@ -0,0 +1,255 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging + +import ( + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + mrpb "google.golang.org/genproto/googleapis/api/monitoredres" +) + +const ( + there = "anyvalue" + projectID = "test-project" + zoneID = "test-region-zone" + regionID = "test-region" + serviceName = "test-service" + version = "1.0" + instanceName = "test-12345" + qualifiedZoneName = "projects/" + projectID + "/zones/" + zoneID + qualifiedRegionName = "projects/" + projectID + "/regions/" + regionID + funcSignature = "test-cf-signature" + funcTarget = "test-cf-target" + crConfig = "test-cr-config" + clusterName = "test-k8s-cluster" + podName = "test-k8s-pod-name" + containerName = "test-k8s-container-name" + namespaceName = "test-k8s-namespace-name" + instanceID = "test-instance-12345" +) + +// fakeResourceGetter mocks internal.ResourceAtttributesGetter interface to retrieve env vars and metadata +type fakeResourceGetter struct { + envVars map[string]string + metaVars map[string]string + fsPaths map[string]string +} + +func (g *fakeResourceGetter) EnvVar(name string) string { + if g.envVars != nil { + if v, ok := g.envVars[name]; ok { + return v + } + } + return "" +} + +func (g *fakeResourceGetter) Metadata(path string) string { + if g.metaVars != nil { + if v, ok := g.metaVars[path]; ok { + return v + } + } + return "" +} + +func (g *fakeResourceGetter) ReadAll(path string) string { + if g.fsPaths != nil { + if v, ok := g.fsPaths[path]; ok { + return v + } + } + return "" +} + +// setupDetectResource resets sync.Once on detectResource and enforces mocked resource attribute getter +func setupDetectedResource(envVars, metaVars, fsPaths map[string]string) { + detectedResource.once = new(sync.Once) + detectedResource.attrs = &fakeResourceGetter{ + envVars: envVars, + metaVars: metaVars, + fsPaths: fsPaths, + } + detectedResource.pb = nil +} + +func TestResourceDetection(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + metaVars map[string]string + fsPaths map[string]string + want *mrpb.MonitoredResource + }{ + { + name: "detect GAE resource", + envVars: map[string]string{"GAE_SERVICE": serviceName, "GAE_VERSION": version, "GAE_INSTANCE": instanceName}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/zone": qualifiedZoneName, "instance/attributes/gae_app_bucket": there}, + want: &mrpb.MonitoredResource{ + Type: "gae_app", + Labels: map[string]string{ + "project_id": projectID, + "module_id": serviceName, + "version_id": version, + "zone": zoneID, + }, + }, + }, + { + name: "detect Cloud Function resource", + envVars: map[string]string{"FUNCTION_TARGET": funcTarget, "FUNCTION_SIGNATURE_TYPE": funcSignature, "K_SERVICE": serviceName}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/region": qualifiedRegionName}, + want: &mrpb.MonitoredResource{ + Type: "cloud_function", + Labels: map[string]string{ + "project_id": projectID, + "region": regionID, + "function_name": serviceName, + }, + }, + }, + { + name: "detect Cloud Run resource", + envVars: map[string]string{"K_CONFIGURATION": crConfig, "K_SERVICE": serviceName, "K_REVISION": version}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/region": qualifiedRegionName}, + want: &mrpb.MonitoredResource{ + Type: "cloud_run_revision", + Labels: map[string]string{ + "project_id": projectID, + "location": regionID, + "service_name": serviceName, + "revision_name": version, + "configuration_name": crConfig, + }, + }, + }, + { + name: "detect GKE resource", + envVars: map[string]string{"HOSTNAME": podName}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/zone": qualifiedZoneName, "instance/attributes/cluster-name": clusterName}, + fsPaths: map[string]string{"/var/run/secrets/kubernetes.io/serviceaccount/namespace": namespaceName}, + want: &mrpb.MonitoredResource{ + Type: "k8s_container", + Labels: map[string]string{ + "cluster_name": clusterName, + "location": zoneID, + "project_id": projectID, + "pod_name": podName, + "namespace_name": namespaceName, + "container_name": "", + }, + }, + }, + { + name: "detect GKE resource with custom container and namespace config", + envVars: map[string]string{"HOSTNAME": podName, "CONTAINER_NAME": containerName, "NAMESPACE_NAME": namespaceName}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/zone": qualifiedZoneName, "instance/attributes/cluster-name": clusterName}, + want: &mrpb.MonitoredResource{ + Type: "k8s_container", + Labels: map[string]string{ + "cluster_name": clusterName, + "location": zoneID, + "project_id": projectID, + "pod_name": podName, + "namespace_name": namespaceName, + "container_name": containerName, + }, + }, + }, + { + name: "detect Compute Engine resource", + envVars: map[string]string{}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/id": instanceID, "instance/zone": qualifiedZoneName, "instance/preempted": there, "instance/cpu-platform": there}, + want: &mrpb.MonitoredResource{ + Type: "gce_instance", + Labels: map[string]string{ + "project_id": projectID, + "instance_id": instanceID, + "zone": zoneID, + }, + }, + }, + { + name: "detect GAE resource by product name", + envVars: map[string]string{}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/zone": qualifiedZoneName, "instance/attributes/gae_app_bucket": there}, + fsPaths: map[string]string{"/sys/class/dmi/id/product_name": "Google App Engine"}, + want: &mrpb.MonitoredResource{ + Type: "gae_app", + Labels: map[string]string{ + "project_id": projectID, + "module_id": "", + "version_id": "", + "zone": zoneID, + }, + }, + }, + { + name: "detect Cloud Function resource by product name", + envVars: map[string]string{}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/region": qualifiedRegionName}, + fsPaths: map[string]string{"/sys/class/dmi/id/product_name": "Google Cloud Functions"}, + want: &mrpb.MonitoredResource{ + Type: "cloud_function", + Labels: map[string]string{ + "project_id": projectID, + "region": regionID, + "function_name": "", + }, + }, + }, + { + name: "detect Cloud Run resource by product name", + envVars: map[string]string{}, + metaVars: map[string]string{"": there, "project/project-id": projectID, "instance/region": qualifiedRegionName}, + fsPaths: map[string]string{"/sys/class/dmi/id/product_name": "Google Cloud Run"}, + want: &mrpb.MonitoredResource{ + Type: "cloud_run_revision", + Labels: map[string]string{ + "project_id": projectID, + "location": regionID, + "service_name": "", + "revision_name": "", + "configuration_name": "", + }, + }, + }, + { + name: "unknown resource detection", + envVars: map[string]string{}, + metaVars: map[string]string{"": there, "project/project-id": projectID}, + want: nil, + }, + { + name: "resource without metadata detection", + envVars: map[string]string{}, + metaVars: map[string]string{}, + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + setupDetectedResource(tc.envVars, tc.metaVars, tc.fsPaths) + got := detectResource() + if diff := cmp.Diff(got, tc.want, cmpopts.IgnoreUnexported(mrpb.MonitoredResource{})); diff != "" { + t.Errorf("got(-),want(+):\n%s", diff) + } + }) + } +}