diff --git a/documentation/src/main/minisite/content/metrics-relay.adoc b/documentation/src/main/minisite/content/metrics-relay.adoc new file mode 100644 index 0000000..8adbe5a --- /dev/null +++ b/documentation/src/main/minisite/content/metrics-relay.adoc @@ -0,0 +1,61 @@ += Getting Started +:minisite-index: 300 +:minisite-index-title: Metrics Relay +:minisite-index-description: Collect batch execution metrics for observability polling. +:minisite-index-icon: heartbeat + +[abstract] +Yupiik Batch provides metrics relay to help components like batch or script to store metrics that will be polled periodically +like Prometheus. +For now the metrics are stored in memory, so it's better to use the dropOnPull to true to avoid too much +memory usage. + +== Stack + +The metrics relay is based on link:https://www.yupiik.io/fusion[Yupiik Fusion]. + +== Usage + +The metrics relay provide an endpoint with 2 methods: + +* POST: `/relay?id=test` to store metrics in OpenMetric format. + +[source,bash] +---- +curl -X POST http://localhost:8080/relay?id=test -H 'Content-Type: text/plain' -d ' +# TYPE foo1 gauge +# HELP foo1 doc +foo1 1234 +# TYPE foo2 gauge +# HELP foo2 doc +foo2 1235' +---- + +The response is a http code 201 with body `OK`. + +You can set the query param `dropOnPull` to `true` (default is `false`) to remove the metric after a polling (GET): + +[source,bash] +---- +curl -X POST http://localhost:8080/relay?id=test&dropOnPull=true -H 'Content-Type: text/plain' -d ' +# TYPE foo1 gauge +# HELP foo1 doc +foo1 1234 +# TYPE foo2 gauge +# HELP foo2 doc +foo2 1235' +---- + +* GET: `/relay?id=test` to fetch the latest metrics. + +[source,bash] +---- +curl http://localhost:8080/relay?id=test +---- + +You can set the query param `ignoreDrop` to `true` (default is `false`) to disable the drop policy define on the entry: + +[source,bash] +---- +curl http://localhost:8080/relay?id=test&ignoreDrop=true +---- diff --git a/pom.xml b/pom.xml index 207ca9c..11fe8a7 100644 --- a/pom.xml +++ b/pom.xml @@ -33,14 +33,14 @@ 1.0.17 1.0.7 1.0.9 - 10.1.15 + 10.1.16 dependencies yupiik-batch-runtime yupiik-batch-ui - yupiik-batch-metrics-scraper + yupiik-batch-metrics-relay documentation iterators simple-configuration @@ -54,6 +54,11 @@ 2.2.220 test + + org.apache.tomcat + tomcat-catalina + ${tomcat.version} + diff --git a/yupiik-batch-metrics-scraper/pom.xml b/yupiik-batch-metrics-relay/pom.xml similarity index 61% rename from yupiik-batch-metrics-scraper/pom.xml rename to yupiik-batch-metrics-relay/pom.xml index f89971c..b144089 100644 --- a/yupiik-batch-metrics-scraper/pom.xml +++ b/yupiik-batch-metrics-relay/pom.xml @@ -24,12 +24,14 @@ 4.0.0 - yupiik-batch-metrics-scraper + yupiik-batch-metrics-relay Yupiik Batch :: Metrics Scraper - ossyupiik/java:17.0.8@sha256:e99a1d0064afd42d0c4d8e2b5a20f10e081c9bb0470485648b01550163a509cc + ossyupiik/java:17.0.9@sha256:88d42b5b803e74fe2149efb942c70d4eb6ecdadb8902c87ac030a01419b2268e + /opt/yupiik scratch + yupiik io.yupiik.fusion.framework.api.main.Launcher @@ -66,7 +68,6 @@ org.apache.tomcat tomcat-catalina - ${tomcat.version} io.yupiik.fusion @@ -135,7 +136,7 @@ 17.0.8-graalce
${main.class}
- yupiik/yupiik-batch/metrics-scraper:${project.version} + ${image.registry}/yupiik-batch/metrics-scraper:${project.version} -1 ${maven.build.timestamp} @@ -144,8 +145,63 @@ ${project.artifactId} ${project.description} ${project.version} - docker run ${image.name} <args> + + -Djava.util.logging.manager=io.yupiik.logging.jul.YupiikLogManager + -Djava.net.preferIPv4Stack=true + -Duser.language=en + -Duser.country=US + -Dfile.encoding=UTF-8 + -H:+UnlockExperimentalVMOptions + --static + +
+ + + + + + docker-jvm + + + + + com.google.cloud.tools + jib-maven-plugin + + packaged + + ${image.java.base} + + + ${image.registry}/yupiik-batch/metrics-scraper:${project.version} + + + ${image.java.workdir} + ${image.java.workdir} + ${image.java.workdir}/yupiik-batch-metrics-relay-${project.version}/lib/* + ${maven.build.timestamp} + ${maven.build.timestamp} + ${main.class} + + -Djava.util.logging.manager=io.yupiik.logging.jul.YupiikLogManager + -Dio.yupiik.logging.jul.handler.StandardHandler.formatter=json + -Djava.security.egd=file:/dev/./urandom + -Djdk.serialFilter=!* + -Dyupiik.build.timestamp=${maven.build.timestamp} + + + ${maven.build.timestamp} + Yupiik + Yupiik + ${project.artifactId} + ${project.description} + ${project.version} + + + + ${project.build.directory}/jib-image.json + diff --git a/yupiik-batch-metrics-scraper/src/main/java/io/yupiik/batch/metricscraper/MetricScraperEndpoint.java b/yupiik-batch-metrics-relay/src/main/java/io/yupiik/batch/metricsrelay/MetricsRelayEndpoint.java similarity index 85% rename from yupiik-batch-metrics-scraper/src/main/java/io/yupiik/batch/metricscraper/MetricScraperEndpoint.java rename to yupiik-batch-metrics-relay/src/main/java/io/yupiik/batch/metricsrelay/MetricsRelayEndpoint.java index fa4c2d1..78452a8 100644 --- a/yupiik-batch-metrics-scraper/src/main/java/io/yupiik/batch/metricscraper/MetricScraperEndpoint.java +++ b/yupiik-batch-metrics-relay/src/main/java/io/yupiik/batch/metricsrelay/MetricsRelayEndpoint.java @@ -13,8 +13,9 @@ * specific language governing permissions and limitations * under the License. */ -package io.yupiik.batch.metricscraper; +package io.yupiik.batch.metricsrelay; +import io.yupiik.fusion.framework.api.scope.ApplicationScoped; import io.yupiik.fusion.framework.build.api.http.HttpMatcher; import io.yupiik.fusion.http.server.api.Request; import io.yupiik.fusion.http.server.api.Response; @@ -25,7 +26,6 @@ import java.io.InputStreamReader; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; -import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.GZIPInputStream; @@ -34,13 +34,14 @@ import static java.util.logging.Level.SEVERE; import static java.util.stream.Collectors.joining; -public class MetricScraperEndpoint { +@ApplicationScoped +public class MetricsRelayEndpoint { - private final static Logger logger = Logger.getLogger(MetricScraperEndpoint.class.getName()); + private final Logger logger = Logger.getLogger(MetricsRelayEndpoint.class.getName()); private final MetricsRelayStorage storage; - public MetricScraperEndpoint(MetricsRelayStorage storage) { + public MetricsRelayEndpoint(MetricsRelayStorage storage) { this.storage = storage; } @@ -51,9 +52,9 @@ public Response storeMetrics(final Request request) { final String id = ofNullable(request.parameter("id")).orElse(""); try (final var in = read(request)) { - storage.store(id, in.lines().collect(joining("\n")).strip(), dropOnPull); + this.storage.store(id, in.lines().collect(joining("\n")).strip(), dropOnPull); } catch (final IOException | ExecutionException | InterruptedException exception) { - Logger.getLogger(getClass().getName()).log(SEVERE, exception, exception::getMessage); + logger.log(SEVERE, exception, exception::getMessage); return Response.of().body("KO").header("Content-Type", responseContentType).build(); } return Response.of().body("OK").status(201).header("Content-Type", responseContentType).build(); @@ -62,7 +63,7 @@ public Response storeMetrics(final Request request) { @HttpMatcher(path = "/relay", methods = "GET", pathMatching = STARTS_WITH) public Response fetchMetrics(final Request request) { final String responseContentType = "application/openmetrics-text; version=1.0.0; charset=utf-8"; - if (storage.getMetrics().isEmpty()) { + if (this.storage.getMetrics().isEmpty()) { return Response.of().header("Content-Type", responseContentType).build(); } @@ -70,7 +71,7 @@ public Response fetchMetrics(final Request request) { final String payload = Stream.of(storage.getMetrics().toArray(new MetricsRelayStorage.Entry[0])) .peek(entry -> { if (!ignoreDrop && entry.dropOnPull()) { - storage.remove(entry); // it is a CopyOnWriteArrayList so it.remove() does not work + this.storage.remove(entry); // it is a CopyOnWriteArrayList so it.remove() does not work } }) .map(MetricsRelayStorage.Entry::content).sorted().collect(joining("\n", "", "\n# EOF\n")); diff --git a/yupiik-batch-metrics-scraper/src/main/java/io/yupiik/batch/metricscraper/MetricsRelayStorage.java b/yupiik-batch-metrics-relay/src/main/java/io/yupiik/batch/metricsrelay/MetricsRelayStorage.java similarity index 97% rename from yupiik-batch-metrics-scraper/src/main/java/io/yupiik/batch/metricscraper/MetricsRelayStorage.java rename to yupiik-batch-metrics-relay/src/main/java/io/yupiik/batch/metricsrelay/MetricsRelayStorage.java index 0481044..f2016bc 100644 --- a/yupiik-batch-metrics-scraper/src/main/java/io/yupiik/batch/metricscraper/MetricsRelayStorage.java +++ b/yupiik-batch-metrics-relay/src/main/java/io/yupiik/batch/metricsrelay/MetricsRelayStorage.java @@ -13,7 +13,7 @@ * specific language governing permissions and limitations * under the License. */ -package io.yupiik.batch.metricscraper; +package io.yupiik.batch.metricsrelay; import io.yupiik.fusion.framework.api.scope.ApplicationScoped; diff --git a/yupiik-batch-metrics-scraper/src/test/java/io/yupiik/batch/metricscraper/EndpointTest.java b/yupiik-batch-metrics-relay/src/test/java/io/yupiik/batch/metricsrelay/EndpointTest.java similarity index 94% rename from yupiik-batch-metrics-scraper/src/test/java/io/yupiik/batch/metricscraper/EndpointTest.java rename to yupiik-batch-metrics-relay/src/test/java/io/yupiik/batch/metricsrelay/EndpointTest.java index 18d58d5..562f6cf 100644 --- a/yupiik-batch-metrics-scraper/src/test/java/io/yupiik/batch/metricscraper/EndpointTest.java +++ b/yupiik-batch-metrics-relay/src/test/java/io/yupiik/batch/metricsrelay/EndpointTest.java @@ -13,7 +13,7 @@ * specific language governing permissions and limitations * under the License. */ -package io.yupiik.batch.metricscraper; +package io.yupiik.batch.metricsrelay; import io.yupiik.fusion.http.server.api.WebServer; import io.yupiik.fusion.testing.Fusion; @@ -22,13 +22,10 @@ import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.logging.Logger; import static java.net.http.HttpResponse.BodyHandlers.ofString; @@ -62,7 +59,7 @@ private HttpResponse sendGetRequest(final WebServer.Configuration config void checkMetrics(@Fusion final WebServer.Configuration configuration) throws IOException, InterruptedException { final var storeMetricResponse = sendPostRequest( configuration, - "/relay?id=test&dropFull=true", + "/relay?id=test&dropOnPull=true", """ # TYPE foo1 gauge # HELP foo1 doc