Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix wrong labels and annotations in kubernetes discovery native client #1282

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
032014e
test
wind57 Dec 4, 2021
c3d0ad2
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Dec 13, 2021
10889fd
fix @Nested tests not running
wind57 Dec 16, 2021
8f0375a
merged main
wind57 Dec 17, 2021
96ebf42
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Jan 5, 2022
db8403e
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Jan 12, 2022
90a5345
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Jan 12, 2022
b041c00
trigger again
wind57 Jan 13, 2022
9580444
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 15, 2022
a34ac47
Add renovate.json
renovate[bot] Jul 13, 2022
6613f78
Merge branch 'spring-cloud:main' into main
wind57 Jul 13, 2022
1b3eae9
Merge pull request #1 from wind57/renovate/configure
wind57 Jul 13, 2022
4275382
Merge branch 'spring-cloud:main' into main
wind57 Mar 13, 2023
618f25a
Delete renovate.json
wind57 Mar 14, 2023
100a9cd
Delete delme.sh
wind57 Mar 14, 2023
4b9056b
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 14, 2023
315a85b
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 15, 2023
ed3c264
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 15, 2023
446d630
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 16, 2023
6821a28
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 16, 2023
9e95a8a
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 16, 2023
02840b1
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 16, 2023
7e31d65
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 20, 2023
926f4d7
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 20, 2023
344e1d4
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 21, 2023
e91ac12
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 23, 2023
ba6e088
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 28, 2023
002c975
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 30, 2023
06c5465
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 30, 2023
63ce5ad
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Mar 31, 2023
335a6f7
dirty
wind57 Apr 1, 2023
49025e8
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Apr 1, 2023
266e7b4
fix issue
wind57 Apr 1, 2023
a647bb3
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Apr 4, 2023
0bf3fd1
before integration test
wind57 Apr 4, 2023
962d350
add integration test
wind57 Apr 4, 2023
16b09ab
Merge branch 'main' of https://github.com/spring-cloud/spring-cloud-k…
wind57 Apr 4, 2023
c6acd76
Merge branch 'main' into fix-wrong-labels-and-annotations
wind57 Apr 4, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,22 @@

package org.springframework.cloud.kubernetes.client.discovery;

import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.openapi.models.V1Service;
import io.kubernetes.client.openapi.models.V1ServiceSpec;
import org.apache.commons.logging.LogFactory;

import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;
import org.springframework.core.log.LogAccessor;

import static org.springframework.cloud.kubernetes.commons.config.ConfigUtils.keysWithPrefix;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.NAMESPACE_METADATA_KEY;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.SERVICE_TYPE;

/**
* @author wind57
*/
Expand Down Expand Up @@ -61,4 +67,38 @@ static boolean matchesServiceLabels(V1Service service, KubernetesDiscoveryProper

}

/**
* This adds the following metadata. <pre>
* - labels (if requested)
* - annotations (if requested)
* - metadata
* - service type
* </pre>
*/
static Map<String, String> serviceMetadata(KubernetesDiscoveryProperties properties, V1Service service,
String serviceId) {

Map<String, String> serviceMetadata = new HashMap<>();
KubernetesDiscoveryProperties.Metadata metadataProps = properties.metadata();
if (metadataProps.addLabels()) {
Map<String, String> labelMetadata = keysWithPrefix(service.getMetadata().getLabels(),
metadataProps.labelsPrefix());
LOG.debug(() -> "Adding labels metadata: " + labelMetadata + " for serviceId: " + serviceId);
serviceMetadata.putAll(labelMetadata);
}
if (metadataProps.addAnnotations()) {
Map<String, String> annotationMetadata = keysWithPrefix(service.getMetadata().getAnnotations(),
metadataProps.annotationsPrefix());
LOG.debug(() -> "Adding annotations metadata: " + annotationMetadata + " for serviceId: " + serviceId);
serviceMetadata.putAll(annotationMetadata);
}

serviceMetadata.put(NAMESPACE_METADATA_KEY,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

metadata now includes two more fields, just like we do in the fabric8 implementation.

Optional.ofNullable(service.getMetadata()).map(V1ObjectMeta::getNamespace).orElse(null));
serviceMetadata.put(SERVICE_TYPE,
Optional.ofNullable(service.getSpec()).map(V1ServiceSpec::getType).orElse(null));

return serviceMetadata;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.springframework.util.StringUtils;

import static org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientUtils.matchesServiceLabels;
import static org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientUtils.serviceMetadata;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.HTTP;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.HTTPS;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.PRIMARY_PORT_NAME_LABEL_KEY;
Expand Down Expand Up @@ -114,27 +115,7 @@ public List<ServiceInstance> getInstances(String serviceId) {
}

private Stream<ServiceInstance> getServiceInstanceDetails(V1Service service, String serviceId) {
Map<String, String> svcMetadata = new HashMap<>();
Copy link
Contributor Author

@wind57 wind57 Apr 1, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this PR fixes an issue, imo. if you look at this piece of code:

service.getMetadata().getAnnotations().entrySet().stream()
							.filter(e -> e.getKey().startsWith(annotationPrefix))
							.forEach(e -> svcMetadata.put(e.getKey(), e.getValue()));

it says that we are going to take only annotations that start with annotationPrefix. In the fabric8 implementation we do it differently: we prefix existing values with it. As a matter of fact we do this in configmaps and secrets property sources also.

As such, I removed this code and fixed the bug in the utils class, where the code is the same as in the fabric8 implementation. Also added tests.

if (properties.metadata() != null) {
if (properties.metadata().addLabels()) {
if (service.getMetadata() != null && service.getMetadata().getLabels() != null) {
String labelPrefix = properties.metadata().labelsPrefix() != null
? properties.metadata().labelsPrefix() : "";
service.getMetadata().getLabels().entrySet().stream()
.filter(e -> e.getKey().startsWith(labelPrefix))
.forEach(e -> svcMetadata.put(e.getKey(), e.getValue()));
}
}
if (properties.metadata().addAnnotations()) {
if (service.getMetadata() != null && service.getMetadata().getAnnotations() != null) {
String annotationPrefix = properties.metadata().annotationsPrefix() != null
? properties.metadata().annotationsPrefix() : "";
service.getMetadata().getAnnotations().entrySet().stream()
.filter(e -> e.getKey().startsWith(annotationPrefix))
.forEach(e -> svcMetadata.put(e.getKey(), e.getValue()));
}
}
}
Map<String, String> serviceMetadata = serviceMetadata(properties, service, serviceId);

V1Endpoints ep = endpointsLister.namespace(service.getMetadata().getNamespace())
.get(service.getMetadata().getName());
Expand All @@ -154,7 +135,7 @@ private Stream<ServiceInstance> getServiceInstanceDetails(V1Service service, Str

return ep.getSubsets().stream().filter(subset -> subset.getPorts() != null && subset.getPorts().size() > 0) // safeguard
.flatMap(subset -> {
Map<String, String> metadata = new HashMap<>(svcMetadata);
Map<String, String> metadata = new HashMap<>(serviceMetadata);
List<CoreV1EndpointPort> endpointPorts = subset.getPorts();
if (properties.metadata() != null && properties.metadata().addPorts()) {
endpointPorts.forEach(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.springframework.cloud.kubernetes.client.discovery;

import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.openapi.models.V1Service;
import io.kubernetes.client.openapi.models.V1ServiceSpec;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;

import org.springframework.boot.test.system.CapturedOutput;
import org.springframework.boot.test.system.OutputCaptureExtension;
import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;

/**
* @author wind57
*/
@ExtendWith(OutputCaptureExtension.class)
class KubernetesDiscoveryClientServiceMetadataTests {

/**
* <pre>
* - labels are not added
* - annotations are not added
* </pre>
*/
@Test
void testServiceMetadataEmpty() {
boolean addLabels = false;
String labelsPrefix = "";
boolean addAnnotations = false;
String annotationsPrefix = "";
boolean addPorts = false;
String portsPrefix = "";

KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(addLabels,
labelsPrefix, addAnnotations, annotationsPrefix, addPorts, portsPrefix);
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L,
true, "", Set.of(), Map.of(), "", metadata, 0, false, false);
V1Service service = new V1Service().spec(new V1ServiceSpec().type("ClusterIP"))
.metadata(new V1ObjectMeta().namespace("default"));

Map<String, String> result = KubernetesDiscoveryClientUtils.serviceMetadata(properties, service, "my-service");
Assertions.assertEquals(result.size(), 2);
Assertions.assertEquals(result, Map.of("k8s_namespace", "default", "type", "ClusterIP"));
}

/**
* <pre>
* - labels are added without a prefix
* - annotations are not added
* </pre>
*/
@Test
void testServiceMetadataAddLabelsNoPrefix(CapturedOutput output) {
boolean addLabels = true;
String labelsPrefix = "";
boolean addAnnotations = false;
String annotationsPrefix = "";
boolean addPorts = false;
String portsPrefix = "";

KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(addLabels,
labelsPrefix, addAnnotations, annotationsPrefix, addPorts, portsPrefix);
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L,
true, "", Set.of(), Map.of(), "", metadata, 0, false, false);
V1Service service = new V1Service().spec(new V1ServiceSpec().type("ClusterIP"))
.metadata(new V1ObjectMeta().namespace("default").labels(Map.of("a", "b")));

Map<String, String> result = KubernetesDiscoveryClientUtils.serviceMetadata(properties, service, "my-service");
Assertions.assertEquals(result.size(), 3);
Assertions.assertEquals(result, Map.of("a", "b", "k8s_namespace", "default", "type", "ClusterIP"));
String labelsMetadata = filterOnK8sNamespaceAndType(result);
Assertions.assertTrue(
output.getOut().contains("Adding labels metadata: " + labelsMetadata + " for serviceId: my-service"));
}

/**
* <pre>
* - labels are added with prefix
* - annotations are not added
* </pre>
*/
@Test
void testServiceMetadataAddLabelsWithPrefix(CapturedOutput output) {
boolean addLabels = true;
String labelsPrefix = "prefix-";
boolean addAnnotations = false;
String annotationsPrefix = "";
boolean addPorts = false;
String portsPrefix = "";

KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(addLabels,
labelsPrefix, addAnnotations, annotationsPrefix, addPorts, portsPrefix);
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L,
true, "", Set.of(), Map.of(), "", metadata, 0, false, false);
V1Service service = new V1Service().spec(new V1ServiceSpec().type("ClusterIP"))
.metadata(new V1ObjectMeta().namespace("default").labels(Map.of("a", "b", "c", "d")));

Map<String, String> result = KubernetesDiscoveryClientUtils.serviceMetadata(properties, service, "my-service");
Assertions.assertEquals(result.size(), 4);
Assertions.assertEquals(result,
Map.of("prefix-a", "b", "prefix-c", "d", "k8s_namespace", "default", "type", "ClusterIP"));
// so that result is deterministic in assertion
String labelsMetadata = filterOnK8sNamespaceAndType(result);
Assertions.assertTrue(
output.getOut().contains("Adding labels metadata: " + labelsMetadata + " for serviceId: my-service"));
}

/**
* <pre>
* - labels are not added
* - annotations are added without prefix
* </pre>
*/
@Test
void testServiceMetadataAddAnnotationsNoPrefix(CapturedOutput output) {
boolean addLabels = false;
String labelsPrefix = "";
boolean addAnnotations = true;
String annotationsPrefix = "";
boolean addPorts = false;
String portsPrefix = "";

KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(addLabels,
labelsPrefix, addAnnotations, annotationsPrefix, addPorts, portsPrefix);
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L,
true, "", Set.of(), Map.of(), "", metadata, 0, false, false);
V1Service service = new V1Service().spec(new V1ServiceSpec().type("ClusterIP")).metadata(
new V1ObjectMeta().namespace("default").labels(Map.of("a", "b")).annotations(Map.of("aa", "bb")));

Map<String, String> result = KubernetesDiscoveryClientUtils.serviceMetadata(properties, service, "my-service");
Assertions.assertEquals(result.size(), 3);
Assertions.assertEquals(result, Map.of("aa", "bb", "k8s_namespace", "default", "type", "ClusterIP"));
Assertions
.assertTrue(output.getOut().contains("Adding annotations metadata: {aa=bb} for serviceId: my-service"));
}

/**
* <pre>
* - labels are not added
* - annotations are added with prefix
* </pre>
*/
@Test
void testServiceMetadataAddAnnotationsWithPrefix(CapturedOutput output) {
boolean addLabels = false;
String labelsPrefix = "";
boolean addAnnotations = true;
String annotationsPrefix = "prefix-";
boolean addPorts = false;
String portsPrefix = "";

KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(addLabels,
labelsPrefix, addAnnotations, annotationsPrefix, addPorts, portsPrefix);
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L,
true, "", Set.of(), Map.of(), "", metadata, 0, false, false);
V1Service service = new V1Service().spec(new V1ServiceSpec().type("ClusterIP")).metadata(new V1ObjectMeta()
.namespace("default").labels(Map.of("a", "b")).annotations(Map.of("aa", "bb", "cc", "dd")));

Map<String, String> result = KubernetesDiscoveryClientUtils.serviceMetadata(properties, service, "my-service");
Assertions.assertEquals(result.size(), 4);
Assertions.assertEquals(result,
Map.of("prefix-aa", "bb", "prefix-cc", "dd", "k8s_namespace", "default", "type", "ClusterIP"));
// so that result is deterministic in assertion
String annotations = filterOnK8sNamespaceAndType(result);
Assertions.assertTrue(
output.getOut().contains("Adding annotations metadata: " + annotations + " for serviceId: my-service"));
}

/**
* <pre>
* - labels are added with prefix
* - annotations are added with prefix
* </pre>
*/
@Test
void testServiceMetadataAddLabelsAndAnnotationsWithPrefix(CapturedOutput output) {
boolean addLabels = true;
String labelsPrefix = "label-";
boolean addAnnotations = true;
String annotationsPrefix = "annotation-";
boolean addPorts = false;
String portsPrefix = "";

KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(addLabels,
labelsPrefix, addAnnotations, annotationsPrefix, addPorts, portsPrefix);
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L,
true, "", Set.of(), Map.of(), "", metadata, 0, false, false);
V1Service service = new V1Service().spec(new V1ServiceSpec().type("ClusterIP")).metadata(new V1ObjectMeta()
.namespace("default").labels(Map.of("a", "b", "c", "d")).annotations(Map.of("aa", "bb", "cc", "dd")));

Map<String, String> result = KubernetesDiscoveryClientUtils.serviceMetadata(properties, service, "my-service");
Assertions.assertEquals(result.size(), 6);
Assertions.assertEquals(result, Map.of("annotation-aa", "bb", "annotation-cc", "dd", "label-a", "b", "label-c",
"d", "k8s_namespace", "default", "type", "ClusterIP"));
// so that result is deterministic in assertion
String labels = result.entrySet().stream().filter(en -> en.getKey().contains("label"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)).toString();
String annotations = result.entrySet().stream().filter(en -> en.getKey().contains("annotation"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)).toString();
Assertions.assertTrue(
output.getOut().contains("Adding labels metadata: " + labels + " for serviceId: my-service"));
Assertions.assertTrue(
output.getOut().contains("Adding annotations metadata: " + annotations + " for serviceId: my-service"));
}

private String filterOnK8sNamespaceAndType(Map<String, String> result) {
return result.entrySet().stream().filter(en -> !en.getKey().contains("k8s_namespace"))
.filter(en -> !en.getKey().equals("type"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)).toString();
}

}
Loading