[modules] Add metrics-scraper documentation

yupiik · Dec 6, 2023 · a7b2a1c · a7b2a1c
1 parent 34e358f
commit a7b2a1c
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 22 deletions.
diff --git a/documentation/src/main/minisite/content/metrics-relay.adoc b/documentation/src/main/minisite/content/metrics-relay.adoc
@@ -0,0 +1,61 @@
+= Getting Started
+:minisite-index: 300
+:minisite-index-title: Metrics Relay
+:minisite-index-description: Collect batch execution metrics for observability polling.
+:minisite-index-icon: heartbeat
+
+[abstract]
+Yupiik Batch provides metrics relay to help components like batch or script to store metrics that will be polled periodically
+like Prometheus.
+For now the metrics are stored in memory, so it's better to use the dropOnPull to true to avoid too much
+memory usage.
+
+== Stack
+
+The metrics relay is based on link:https://www.yupiik.io/fusion[Yupiik Fusion].
+
+== Usage
+
+The metrics relay provide an endpoint with 2 methods:
+
+* POST: `/relay?id=test` to store metrics in OpenMetric format.
+
+[source,bash]
+----
+curl -X POST http://localhost:8080/relay?id=test -H 'Content-Type: text/plain' -d '
+# TYPE foo1 gauge
+# HELP foo1 doc
+foo1 1234
+# TYPE foo2 gauge
+# HELP foo2 doc
+foo2 1235'
+----
+
+The response is a http code 201 with body `OK`.
+
+You can set the query param `dropOnPull` to `true` (default is `false`) to remove the metric after a polling (GET):
+
+[source,bash]
+----
+curl -X POST http://localhost:8080/relay?id=test&dropOnPull=true -H 'Content-Type: text/plain' -d '
+# TYPE foo1 gauge
+# HELP foo1 doc
+foo1 1234
+# TYPE foo2 gauge
+# HELP foo2 doc
+foo2 1235'
+----
+
+* GET: `/relay?id=test` to fetch the latest metrics.
+
+[source,bash]
+----
+curl http://localhost:8080/relay?id=test
+----
+
+You can set the query param `ignoreDrop` to `true` (default is `false`) to disable the drop policy define on the entry:
+
+[source,bash]
+----
+curl http://localhost:8080/relay?id=test&ignoreDrop=true
+----
diff --git a/pom.xml b/pom.xml
@@ -33,14 +33,14 @@
     <uship.version>1.0.17</uship.version>
     <yupiik-logging.version>1.0.7</yupiik-logging.version>
     <fusion.version>1.0.9</fusion.version>
-    <tomcat.version>10.1.15</tomcat.version>
+    <tomcat.version>10.1.16</tomcat.version>
   </properties>
 
   <modules>
     <module>dependencies</module>
     <module>yupiik-batch-runtime</module>
     <module>yupiik-batch-ui</module>
-    <module>yupiik-batch-metrics-scraper</module>
+    <module>yupiik-batch-metrics-relay</module>
     <module>documentation</module>
     <module>iterators</module>
     <module>simple-configuration</module>
@@ -54,6 +54,11 @@
         <version>2.2.220</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.apache.tomcat</groupId>
+        <artifactId>tomcat-catalina</artifactId>
+        <version>${tomcat.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 

diff --git a/yupiik-batch-metrics-scraper/pom.xml → yupiik-batch-metrics-relay/pom.xml b/yupiik-batch-metrics-scraper/pom.xml → yupiik-batch-metrics-relay/pom.xml
@@ -24,12 +24,14 @@
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
-  <artifactId>yupiik-batch-metrics-scraper</artifactId>
+  <artifactId>yupiik-batch-metrics-relay</artifactId>
   <name>Yupiik Batch :: Metrics Scraper</name>
 
   <properties>
-    <image.java.base>ossyupiik/java:17.0.8@sha256:e99a1d0064afd42d0c4d8e2b5a20f10e081c9bb0470485648b01550163a509cc</image.java.base>
+    <image.java.base>ossyupiik/java:17.0.9@sha256:88d42b5b803e74fe2149efb942c70d4eb6ecdadb8902c87ac030a01419b2268e</image.java.base>
+    <image.java.workdir>/opt/yupiik</image.java.workdir>
     <image.native.base>scratch</image.native.base>
+    <image.registry>yupiik</image.registry>
     <main.class>io.yupiik.fusion.framework.api.main.Launcher</main.class>
   </properties>
 
@@ -66,7 +68,6 @@
     <dependency>
       <groupId>org.apache.tomcat</groupId>
       <artifactId>tomcat-catalina</artifactId>
-      <version>${tomcat.version}</version>
     </dependency>
     <dependency>
       <groupId>io.yupiik.fusion</groupId>
@@ -135,7 +136,7 @@
             <configuration>
               <graalVersion>17.0.8-graalce</graalVersion>
               <main>${main.class}</main>
-              <to>yupiik/yupiik-batch/metrics-scraper:${project.version}</to>
+              <to>${image.registry}/yupiik-batch/metrics-scraper:${project.version}</to>
               <creationTimestamp>-1</creationTimestamp>
               <labels>
                 <org.opencontainers.image.created>${maven.build.timestamp}</org.opencontainers.image.created>
@@ -144,8 +145,63 @@
                 <org.opencontainers.image.title>${project.artifactId}</org.opencontainers.image.title>
                 <org.opencontainers.image.description>${project.description}</org.opencontainers.image.description>
                 <org.opencontainers.image.version>${project.version}</org.opencontainers.image.version>
-                <io.yupiik.cmd>docker run ${image.name} &lt;args&gt;</io.yupiik.cmd>
               </labels>
+              <customOptions>
+                <customOption>-Djava.util.logging.manager=io.yupiik.logging.jul.YupiikLogManager</customOption>
+                <customOption>-Djava.net.preferIPv4Stack=true</customOption>
+                <customOption>-Duser.language=en</customOption>
+                <customOption>-Duser.country=US</customOption>
+                <customOption>-Dfile.encoding=UTF-8</customOption>
+                <customOption>-H:+UnlockExperimentalVMOptions</customOption>
+                <customOption>--static</customOption>
+              </customOptions>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>docker-jvm</id>
+      <build>
+        <plugins>
+          <!-- mvn jib:docker -Pdocker-jvm -->
+          <plugin>
+            <groupId>com.google.cloud.tools</groupId>
+            <artifactId>jib-maven-plugin</artifactId>
+            <configuration>
+              <containerizingMode>packaged</containerizingMode>
+              <from>
+                <image>${image.java.base}</image>
+              </from>
+              <to>
+                <image>${image.registry}/yupiik-batch/metrics-scraper:${project.version}</image>
+              </to>
+              <container>
+                <appRoot>${image.java.workdir}</appRoot>
+                <workingDirectory>${image.java.workdir}</workingDirectory>
+                <extraClasspath>${image.java.workdir}/yupiik-batch-metrics-relay-${project.version}/lib/*</extraClasspath>
+                <creationTime>${maven.build.timestamp}</creationTime>
+                <filesModificationTime>${maven.build.timestamp}</filesModificationTime>
+                <mainClass>${main.class}</mainClass>
+                <jvmFlags>
+                  <jvmFlag>-Djava.util.logging.manager=io.yupiik.logging.jul.YupiikLogManager</jvmFlag>
+                  <jvmFlag>-Dio.yupiik.logging.jul.handler.StandardHandler.formatter=json</jvmFlag>
+                  <jvmFlag>-Djava.security.egd=file:/dev/./urandom</jvmFlag>
+                  <jvmFlag>-Djdk.serialFilter=!*</jvmFlag>
+                  <jvmFlag>-Dyupiik.build.timestamp=${maven.build.timestamp}</jvmFlag>
+                </jvmFlags>
+                <labels>
+                  <org.opencontainers.image.created>${maven.build.timestamp}</org.opencontainers.image.created>
+                  <org.opencontainers.image.authors>Yupiik</org.opencontainers.image.authors>
+                  <org.opencontainers.image.vendor>Yupiik</org.opencontainers.image.vendor>
+                  <org.opencontainers.image.title>${project.artifactId}</org.opencontainers.image.title>
+                  <org.opencontainers.image.description>${project.description}</org.opencontainers.image.description>
+                  <org.opencontainers.image.version>${project.version}</org.opencontainers.image.version>
+                </labels>
+              </container>
+              <outputPaths>
+                <imageJson>${project.build.directory}/jib-image.json</imageJson>
+              </outputPaths>
             </configuration>
           </plugin>
         </plugins>

diff --git a/.../metricscraper/MetricScraperEndpoint.java → ...ch/metricsrelay/MetricsRelayEndpoint.java b/.../metricscraper/MetricScraperEndpoint.java → ...ch/metricsrelay/MetricsRelayEndpoint.java
@@ -13,8 +13,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package io.yupiik.batch.metricscraper;
+package io.yupiik.batch.metricsrelay;
 
+import io.yupiik.fusion.framework.api.scope.ApplicationScoped;
 import io.yupiik.fusion.framework.build.api.http.HttpMatcher;
 import io.yupiik.fusion.http.server.api.Request;
 import io.yupiik.fusion.http.server.api.Response;
@@ -25,7 +26,6 @@
 import java.io.InputStreamReader;
 import java.util.concurrent.ExecutionException;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 
@@ -34,13 +34,14 @@
 import static java.util.logging.Level.SEVERE;
 import static java.util.stream.Collectors.joining;
 
-public class MetricScraperEndpoint {
+@ApplicationScoped
+public class MetricsRelayEndpoint {
 
-    private final static Logger logger = Logger.getLogger(MetricScraperEndpoint.class.getName());
+    private final Logger logger = Logger.getLogger(MetricsRelayEndpoint.class.getName());
 
     private final MetricsRelayStorage storage;
 
-    public MetricScraperEndpoint(MetricsRelayStorage storage) {
+    public MetricsRelayEndpoint(MetricsRelayStorage storage) {
         this.storage = storage;
     }
 
@@ -51,9 +52,9 @@ public Response storeMetrics(final Request request) {
         final String id = ofNullable(request.parameter("id")).orElse("");
 
         try (final var in = read(request)) {
-            storage.store(id, in.lines().collect(joining("\n")).strip(), dropOnPull);
+            this.storage.store(id, in.lines().collect(joining("\n")).strip(), dropOnPull);
         } catch (final IOException | ExecutionException | InterruptedException exception) {
-            Logger.getLogger(getClass().getName()).log(SEVERE, exception, exception::getMessage);
+            logger.log(SEVERE, exception, exception::getMessage);
             return Response.of().body("KO").header("Content-Type", responseContentType).build();
         }
         return Response.of().body("OK").status(201).header("Content-Type", responseContentType).build();
@@ -62,15 +63,15 @@ public Response storeMetrics(final Request request) {
     @HttpMatcher(path = "/relay", methods = "GET", pathMatching = STARTS_WITH)
     public Response fetchMetrics(final Request request) {
         final String responseContentType = "application/openmetrics-text; version=1.0.0; charset=utf-8";
-        if (storage.getMetrics().isEmpty()) {
+        if (this.storage.getMetrics().isEmpty()) {
             return Response.of().header("Content-Type", responseContentType).build();
         }
 
         final boolean ignoreDrop = Boolean.parseBoolean(ofNullable(request.parameter("ignoreDrop")).orElse("false"));
         final String payload = Stream.of(storage.getMetrics().toArray(new MetricsRelayStorage.Entry[0]))
                 .peek(entry -> {
                     if (!ignoreDrop && entry.dropOnPull()) {
-                        storage.remove(entry); // it is a CopyOnWriteArrayList so it.remove() does not work
+                        this.storage.remove(entry); // it is a CopyOnWriteArrayList so it.remove() does not work
                     }
                 })
                 .map(MetricsRelayStorage.Entry::content).sorted().collect(joining("\n", "", "\n# EOF\n"));

diff --git a/...ch/metricscraper/MetricsRelayStorage.java → ...tch/metricsrelay/MetricsRelayStorage.java b/...ch/metricscraper/MetricsRelayStorage.java → ...tch/metricsrelay/MetricsRelayStorage.java
@@ -13,7 +13,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package io.yupiik.batch.metricscraper;
+package io.yupiik.batch.metricsrelay;
 
 import io.yupiik.fusion.framework.api.scope.ApplicationScoped;
 

diff --git a/...iik/batch/metricscraper/EndpointTest.java → ...piik/batch/metricsrelay/EndpointTest.java b/...iik/batch/metricscraper/EndpointTest.java → ...piik/batch/metricsrelay/EndpointTest.java
@@ -13,7 +13,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package io.yupiik.batch.metricscraper;
+package io.yupiik.batch.metricsrelay;
 
 import io.yupiik.fusion.http.server.api.WebServer;
 import io.yupiik.fusion.testing.Fusion;
@@ -22,13 +22,10 @@
 
 import java.io.IOException;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.logging.Logger;
 
 import static java.net.http.HttpResponse.BodyHandlers.ofString;
@@ -62,7 +59,7 @@ private HttpResponse<String> sendGetRequest(final WebServer.Configuration config
     void checkMetrics(@Fusion final WebServer.Configuration configuration) throws IOException, InterruptedException {
         final var storeMetricResponse = sendPostRequest(
                 configuration,
-                "/relay?id=test&dropFull=true",
+                "/relay?id=test&dropOnPull=true",
                 """
                 # TYPE foo1 gauge
                 # HELP foo1 doc