Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor the OTelSdkProvider into an implementation of OpenTelemetry #704

Merged
merged 21 commits into from
Jun 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
b3556b2
Cleanup OTelSdkProvider
cyrille-leclerc Sep 15, 2023
2394b80
merge master
cyrille-leclerc Sep 15, 2023
55a78df
Cleanup OTelSdkProvider
cyrille-leclerc Sep 18, 2023
5eb9f7e
Merge branch 'master' into cleanup-otel-provider
cyrille-leclerc Sep 18, 2023
c4e88e8
Cleanup OTelSdkProvider
cyrille-leclerc Sep 18, 2023
d350353
Merge branch 'master' into cleanup-otel-provider
cyrille-leclerc Sep 24, 2023
16f15ad
Merge branch 'master' into cleanup-otel-provider
cyrille-leclerc Oct 3, 2023
db9f6b1
merge master branch
cyrille-leclerc Oct 27, 2023
4563cb5
Merge branch 'master' into cleanup-otel-provider
cyrille-leclerc Oct 27, 2023
5094187
Cleanup OTelSdkProvider
cyrille-leclerc Oct 29, 2023
0fbe955
Cleanup OTelSdkProvider
cyrille-leclerc Nov 9, 2023
f80b493
Merge branch 'master' into cleanup-otel-provider
cyrille-leclerc Dec 22, 2023
4001b38
Merge master
cyrille-leclerc Jan 2, 2024
d6ab809
Merge master
cyrille-leclerc Jan 2, 2024
a67e5cf
Merge branch 'main' into cleanup-otel-provider
cyrille-leclerc Jan 28, 2024
03028bd
Merge branch 'main' into cleanup-otel-provider
cyrille-leclerc Mar 13, 2024
1c8693d
Merge branch 'main' into cleanup-otel-provider
cyrille-leclerc May 22, 2024
9def781
Merge branch 'main' into cleanup-otel-provider
cyrille-leclerc Jun 5, 2024
34102f2
Merge branch 'main' into cleanup-otel-provider
cyrille-leclerc Jun 17, 2024
a702d36
Merge branch 'main' into cleanup-otel-provider
cyrille-leclerc Jun 18, 2024
d0c9ca4
Add reconfigurability of the TraceProvider, LoggerProvider, and Even…
cyrille-leclerc Jun 20, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
<gitHubRepo>jenkinsci/${project.artifactId}-plugin</gitHubRepo>
<opentelemetry.version>1.39.0</opentelemetry.version>
<opentelemetry-instrumentation.version>2.5.0</opentelemetry-instrumentation.version>
<opentelemetry-semconv.version>1.25.0-alpha</opentelemetry-semconv.version>
<opentelemetry-contrib.version>1.36.0-alpha</opentelemetry-contrib.version>
<useBeta>true</useBeta>
<elasticstack.version>8.14.1</elasticstack.version>
Expand Down Expand Up @@ -93,15 +94,32 @@
</dependencyManagement>
<dependencies>
<dependency>
<groupId>io.jenkins.plugins</groupId>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-api</artifactId>
<version>1.39.0-8.vfb_39d89a_2812</version>
</dependency>

<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-api-incubator</artifactId>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-context</artifactId>
</dependency>
<dependency>
<groupId>io.opentelemetry.semconv</groupId>
<artifactId>opentelemetry-semconv</artifactId>
<version>${opentelemetry-semconv.version}</version>
</dependency>
<dependency>
<groupId>io.opentelemetry.semconv</groupId>
<artifactId>opentelemetry-semconv-incubating</artifactId>
<version>${opentelemetry-semconv.version}</version>
</dependency>
<dependency>
<groupId>io.opentelemetry.instrumentation</groupId>
<artifactId>opentelemetry-instrumentation-api</artifactId>
</dependency>

<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk</artifactId>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Copyright The Original Author or Authors
* SPDX-License-Identifier: Apache-2.0
*/

package io.jenkins.plugins.opentelemetry;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.ExtensionList;
import io.jenkins.plugins.opentelemetry.opentelemetry.ReconfigurableOpenTelemetry;
import io.jenkins.plugins.opentelemetry.semconv.JenkinsOtelSemanticAttributes;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.incubator.events.EventLogger;
import io.opentelemetry.api.incubator.events.GlobalEventLoggerProvider;
import io.opentelemetry.api.metrics.Meter;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.instrumentation.resources.ProcessResourceProvider;
import io.opentelemetry.sdk.OpenTelemetrySdk;

import javax.annotation.PreDestroy;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;

/**
* {@link OpenTelemetry} instance intended to live on the Jenkins Controller.
*/
@Extension
public class JenkinsControllerOpenTelemetry extends ReconfigurableOpenTelemetry implements OpenTelemetry {

private static final Logger LOGGER = Logger.getLogger(JenkinsControllerOpenTelemetry.class.getName());

/**
* See {@code OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS}
*/
public static final String DEFAULT_OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS = ProcessResourceProvider.class.getName();

public final static AtomicInteger INSTANCE_COUNTER = new AtomicInteger(0);

@NonNull
private final transient Tracer defaultTracer;
protected transient Meter defaultMeter;
protected final transient EventLogger defaultEventLogger;

public JenkinsControllerOpenTelemetry() {
super();
if (INSTANCE_COUNTER.get() > 0) {

Check warning on line 51 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsControllerOpenTelemetry.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 51 is only partially covered, one branch is missing
LOGGER.log(Level.WARNING, "More than one instance of JenkinsControllerOpenTelemetry created: " + INSTANCE_COUNTER.get());

Check warning on line 52 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsControllerOpenTelemetry.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line

Line 52 is not covered by tests
}

String opentelemetryPluginVersion = OtelUtils.getOpentelemetryPluginVersion();

this.defaultTracer =
getTracerProvider()
.tracerBuilder(JenkinsOtelSemanticAttributes.INSTRUMENTATION_NAME)
.setInstrumentationVersion(opentelemetryPluginVersion)
.build();

this.defaultEventLogger = getEventLoggerProvider()
.eventLoggerBuilder(JenkinsOtelSemanticAttributes.INSTRUMENTATION_NAME)
.setInstrumentationVersion(opentelemetryPluginVersion)
.build();
}

@NonNull
public Tracer getDefaultTracer() {
return defaultTracer;
}

public boolean isLogsEnabled() {
String otelLogsExporter = config.getString("otel.logs.exporter");
return otelLogsExporter != null && !otelLogsExporter.equals("none");

Check warning on line 76 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsControllerOpenTelemetry.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 76 is only partially covered, one branch is missing
}

public boolean isOtelLogsMirrorToDisk() {
String otelLogsExporter = config.getString("otel.logs.mirror_to_disk");
return otelLogsExporter != null && otelLogsExporter.equals("true");

Check warning on line 81 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsControllerOpenTelemetry.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 81 is only partially covered, one branch is missing
}

@VisibleForTesting
@NonNull
protected OpenTelemetrySdk getOpenTelemetrySdk() {
Preconditions.checkNotNull(getOpenTelemetryDelegate());
if (getOpenTelemetryDelegate() instanceof OpenTelemetrySdk) {

Check warning on line 88 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsControllerOpenTelemetry.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 88 is only partially covered, one branch is missing
return (OpenTelemetrySdk) getOpenTelemetryDelegate();
} else {
throw new IllegalStateException("OpenTelemetry initialized as NoOp");

Check warning on line 91 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsControllerOpenTelemetry.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line

Line 91 is not covered by tests
}
}

@PreDestroy
public void shutdown() {
super.close();
}

public void initialize(@NonNull OpenTelemetryConfiguration configuration) {
configure(
configuration.toOpenTelemetryProperties(),
configuration.toOpenTelemetryResource());
}

@Override
protected void postOpenTelemetrySdkConfiguration() {
String opentelemetryPluginVersion = OtelUtils.getOpentelemetryPluginVersion();

this.defaultMeter = getMeterProvider()
.meterBuilder(JenkinsOtelSemanticAttributes.INSTRUMENTATION_NAME)
.setInstrumentationVersion(opentelemetryPluginVersion)
.build();

LOGGER.log(Level.FINER, () -> "Configure OpenTelemetryLifecycleListeners: " + ExtensionList.lookup(OpenTelemetryLifecycleListener.class).stream().sorted().map(e -> e.getClass().getName()).collect(Collectors.joining(", ")));
ExtensionList.lookup(OpenTelemetryLifecycleListener.class).stream()
.sorted()
.forEachOrdered(otelComponent -> {
otelComponent.afterSdkInitialized(defaultMeter, getOpenTelemetryDelegate().getLogsBridge(), defaultEventLogger, defaultTracer, config);
otelComponent.afterSdkInitialized(getOpenTelemetryDelegate(), config);
});
}

static public JenkinsControllerOpenTelemetry get() {
return ExtensionList.lookupSingleton(JenkinsControllerOpenTelemetry.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,9 @@

private String ignoredSteps = "dir,echo,isUnix,pwd,properties";

private String disabledResourceProviders = OpenTelemetrySdkProvider.DEFAULT_OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS;
private String disabledResourceProviders = JenkinsControllerOpenTelemetry.DEFAULT_OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS;

private transient OpenTelemetrySdkProvider openTelemetrySdkProvider;
private transient JenkinsControllerOpenTelemetry jenkinsControllerOpenTelemetry;

private transient LogStorageRetriever logStorageRetriever;

Expand Down Expand Up @@ -182,12 +182,11 @@
protected Object readResolve() {
LOGGER.log(Level.FINE, "readResolve()");
if (this.disabledResourceProviders == null) {
this.disabledResourceProviders = OpenTelemetrySdkProvider.DEFAULT_OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS;
this.disabledResourceProviders = JenkinsControllerOpenTelemetry.DEFAULT_OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS;

Check warning on line 185 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsOpenTelemetryPluginConfiguration.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line

Line 185 is not covered by tests
}
return this;
}


@NonNull
public OpenTelemetryConfiguration toOpenTelemetryConfiguration() {
Properties properties = new Properties();
Expand All @@ -197,7 +196,7 @@
LOGGER.log(Level.WARNING, "Exception parsing configuration properties", e);
}

Map<String, String> configurationProperties = new HashMap();
Map<String, String> configurationProperties = new HashMap<>();
getObservabilityBackends().forEach(backend -> configurationProperties.putAll(backend.getOtelConfigurationProperties()));
configurationProperties.put(JenkinsOtelSemanticAttributes.JENKINS_VERSION.getKey(), OtelUtils.getJenkinsVersion());
configurationProperties.put(JenkinsOtelSemanticAttributes.JENKINS_URL.getKey(), this.jenkinsLocationConfiguration.getUrl());
Expand All @@ -209,7 +208,7 @@
return new OpenTelemetryConfiguration(
Optional.ofNullable(this.getEndpoint()),
Optional.ofNullable(this.getTrustedCertificatesPem()),
Optional.ofNullable(this.getAuthentication()),
Optional.of(this.getAuthentication()),
Optional.ofNullable(this.getExporterTimeoutMillis()),
Optional.ofNullable(this.getExporterIntervalMillis()),
Optional.ofNullable(this.getServiceName()),
Expand All @@ -218,19 +217,33 @@
configurationProperties);
}

/**
* Register reconfigurable {@link io.opentelemetry.api.OpenTelemetry}
* on {@link io.opentelemetry.api.GlobalOpenTelemetry}
* and {@link io.jenkins.plugins.opentelemetry.opentelemetry.ReconfigurableEventLoggerProvider}
* on {@link io.opentelemetry.api.incubator.events.GlobalEventLoggerProvider}
* as early as possible in Jenkins lifecycle so any plugin invoking those Global setters will have the
* reconfigurable instance .
*/
@Initializer(after = InitMilestone.EXTENSIONS_AUGMENTED, before = InitMilestone.SYSTEM_CONFIG_LOADED)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

before system config loaded? how will the configuration be available for this plugin? and how will someone configure this plugin with JCasC if its so early?

jcasc is after system config loaded:
https://github.com/jenkinsci/configuration-as-code-plugin/blob/3b0e6de4ab83e8024e6cac635728c28b68ac663a/plugin/src/main/java/io/jenkins/plugins/casc/ConfigurationAsCode.java#L337

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct: There reasons is:

  1. we want to be sure that any plugin that is using OTel will use our implementation of OpenTelemetry, even when getting this OpenTelemetry instance through GlobalOpenTelemetry.get()
  2. we don't know how to ensure the OpenTelemetry plugin gets configured & initialized before all other plugins (e.g. the initialization code DatabaseSchemaLoader#migrateSchema() of the JUnit SQL Storage Plugin is invoked by Jenkins before the Jenkins OTel Plugin initialization code JenkinsOpenTelemetryPluginConfiguration#initializeOpenTelemetry() despite the dependency.

The solution we identified is to:

  1. Register a Reconfigurable OpenTelemetry instance in GlobalOpenTelemetry as early as possible in Jenkin's lifecycle with a NoOp implementation (see initializeOpenTelemetryAfterExtensionsAugmented()).
  2. When the Jenkins Otel Plugin config is ready, we reconfigure the OpenTelmetry instance (see initializeOpenTelemetry())

I think we can cleanup a bit this code but it works and it was quite time consuming to get there :-)

Dos it make sense?

https://github.com/jenkinsci/junit-sql-storage-plugin/blob/76cefd5f4cf6117553fa6f219442a9cbba64b458/src/main/java/io/jenkins/plugins/junit/storage/database/DatabaseSchemaLoader.java#L24-L26

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah I see bit confusing but it'll work I think

public void initializeOpenTelemetryAfterExtensionsAugmented() {
LOGGER.log(Level.INFO, "Initialize Jenkins OpenTelemetry Plugin with a NoOp implementation...");
jenkinsControllerOpenTelemetry.configure(Collections.emptyMap(), Resource.empty());
}

/**
* Initialize the Otel SDK, must happen after the plugin has been configured by the standard config and by JCasC
* JCasC configuration happens during `SYSTEM_CONFIG_ADAPTED` (see `io.jenkins.plugins.casc.ConfigurationAsCode#init()`)
*/
@Initializer(after = InitMilestone.SYSTEM_CONFIG_ADAPTED, before = InitMilestone.JOB_LOADED)
@SuppressWarnings("MustBeClosedChecker")
public void initializeOpenTelemetry() {
LOGGER.log(Level.FINE, "Initialize Jenkins OpenTelemetry Plugin...");
LOGGER.log(Level.INFO, "Initialize Jenkins OpenTelemetry Plugin...");
OpenTelemetryConfiguration newOpenTelemetryConfiguration = toOpenTelemetryConfiguration();
if (Objects.equals(this.currentOpenTelemetryConfiguration, newOpenTelemetryConfiguration)) {
LOGGER.log(Level.FINE, "Configuration didn't change, skip reconfiguration");
} else {
openTelemetrySdkProvider.initialize(newOpenTelemetryConfiguration);
jenkinsControllerOpenTelemetry.initialize(newOpenTelemetryConfiguration);
this.currentOpenTelemetryConfiguration = newOpenTelemetryConfiguration;
}

Expand Down Expand Up @@ -308,8 +321,8 @@
}

@Inject
public void setOpenTelemetrySdkProvider(OpenTelemetrySdkProvider openTelemetrySdkProvider) {
this.openTelemetrySdkProvider = openTelemetrySdkProvider;
public void setJenkinsControllerOpenTelemetry(JenkinsControllerOpenTelemetry jenkinsControllerOpenTelemetry) {
this.jenkinsControllerOpenTelemetry = jenkinsControllerOpenTelemetry;
}

public Integer getExporterTimeoutMillis() {
Expand Down Expand Up @@ -506,7 +519,7 @@
}

/**
* @see ServiceIncubatingAttributes#SERVICE_NAMESPACE
* @see io.opentelemetry.semconv.incubating.ServiceIncubatingAttributes#SERVICE_NAMESPACE
*/
public String getServiceNamespace() {
return (Strings.isNullOrEmpty(this.serviceNamespace)) ? JenkinsOtelSemanticAttributes.JENKINS : this.serviceNamespace;
Expand All @@ -520,30 +533,30 @@

@NonNull
public Resource getResource() {
if (this.openTelemetrySdkProvider == null) {
if (this.jenkinsControllerOpenTelemetry == null) {
return Resource.empty();
} else {
return this.openTelemetrySdkProvider.getResource();
return this.jenkinsControllerOpenTelemetry.getResource();
}
}

/**
* Used in io/jenkins/plugins/opentelemetry/JenkinsOpenTelemetryPluginConfiguration/config.jelly because
* cyrille doesn't know how to format the content with linebreaks in a html teaxtarea
*/
@NonNull
public String getResourceAsText() {
return this.getResource().getAttributes().asMap().entrySet().stream().
map(e -> e.getKey() + "=" + e.getValue()).
collect(Collectors.joining("\r\n"));
}

@NonNull
public ConfigProperties getConfigProperties() {
if (this.openTelemetrySdkProvider == null) {
if (this.jenkinsControllerOpenTelemetry == null) {
return ConfigPropertiesUtils.emptyConfig();
} else {
return this.openTelemetrySdkProvider.getConfig();
return this.jenkinsControllerOpenTelemetry.getConfig();

Check warning on line 559 in src/main/java/io/jenkins/plugins/opentelemetry/JenkinsOpenTelemetryPluginConfiguration.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 536-559 are not covered by tests
}
}

Expand All @@ -568,11 +581,12 @@

@NonNull
@MustBeClosed
@SuppressWarnings("MustBeClosedChecker") // false positive invoking backend.getLogStorageRetriever(templateBindingsProvider)
@SuppressWarnings("MustBeClosedChecker")
// false positive invoking backend.getLogStorageRetriever(templateBindingsProvider)
private LogStorageRetriever resolveLogStorageRetriever() {
LogStorageRetriever logStorageRetriever = null;

Resource otelSdkResource = openTelemetrySdkProvider.getResource();
Resource otelSdkResource = jenkinsControllerOpenTelemetry.getResource();
String serviceName = Objects.requireNonNull(otelSdkResource.getAttribute(ServiceAttributes.SERVICE_NAME), "service.name can't be null");
String serviceNamespace = otelSdkResource.getAttribute(ServiceIncubatingAttributes.SERVICE_NAMESPACE);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
* <p>
* Used by components that create counters...
*/
public interface OtelComponent extends Comparable<OtelComponent>{
public interface OpenTelemetryLifecycleListener extends Comparable<OpenTelemetryLifecycleListener>{

/**
* Invoked soon after the Otel SDK has been initialized.
Expand Down Expand Up @@ -64,7 +64,7 @@ default int ordinal() {
}

@Override
default int compareTo(OtelComponent other) {
default int compareTo(OpenTelemetryLifecycleListener other) {
if (this.ordinal() == other.ordinal()) {
return this.getClass().getName().compareTo(other.getClass().getName());
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@
import java.util.logging.Level;
import java.util.logging.Logger;

/**
* Decorates Jenkins navigation GUI with the OpenTelemetry dashboard link if defined
*/
@Extension
public class OpenTelemetryRootAction implements RootAction {
private static final Logger logger = Logger.getLogger(OpenTelemetryRootAction.class.getName());

private JenkinsOpenTelemetryPluginConfiguration pluginConfiguration;
private OpenTelemetrySdkProvider openTelemetrySdkProvider;
private JenkinsControllerOpenTelemetry jenkinsControllerOpenTelemetry;

public Optional<ObservabilityBackend> getFirstMetricsCapableObservabilityBackend() {
final Optional<ObservabilityBackend> observabilityBackend = pluginConfiguration.getObservabilityBackends()
Expand All @@ -45,9 +48,9 @@

@Override
public String getUrlName() {
// TODO we could keep in cache this URL

Check warning on line 51 in src/main/java/io/jenkins/plugins/opentelemetry/OpenTelemetryRootAction.java

View check run for this annotation

ci.jenkins.io / Open Tasks Scanner

TODO

NORMAL: we could keep in cache this URL
return getFirstMetricsCapableObservabilityBackend()
.map(backend -> backend.getMetricsVisualizationUrl(this.openTelemetrySdkProvider.getResource()))
.map(backend -> backend.getMetricsVisualizationUrl(this.jenkinsControllerOpenTelemetry.getResource()))
.orElse(null);
}

Expand All @@ -57,7 +60,7 @@
}

@Inject
public void setOpenTelemetrySdkProvider(OpenTelemetrySdkProvider openTelemetrySdkProvider) {
this.openTelemetrySdkProvider = openTelemetrySdkProvider;
public void setJenkinsControllerOpenTelemetry(JenkinsControllerOpenTelemetry jenkinsControllerOpenTelemetry) {
this.jenkinsControllerOpenTelemetry = jenkinsControllerOpenTelemetry;
}
}
Loading
Loading