output server and scheduler logs to files (#1427)

* output server and scheduler logs to files * add workspace root env variable to docker compose up * add workspace root to kubernetes as well
airbytehq · Dec 28, 2020 · c990ec6 · c990ec6
1 parent 26241aa
commit c990ec6
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 0 deletions.
diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml
@@ -23,12 +23,37 @@
             </Routes>
             <IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
         </Routing>
+        <Routing name="AppLogSplit">
+            <Routes pattern="$${ctx:workspace_app_root}">
+                <!-- Don't split logs if workspace_app_log_root isn't defined -->
+                <Route key="$${ctx:workspace_app_root}">
+                    <Null name="/dev/null"/>
+                </Route>
+                <Route>
+                    <RollingFile
+                            name="${ctx:workspace_app_root}"
+                            fileName="${ctx:workspace_app_root}/logs.log"
+                            filePattern="${ctx:workspace_app_root}/logs.%i.log.gz"
+                            ignoreExceptions="false">
+                        <PatternLayout>
+                            <Pattern>%d{yyyy-MM-dd HH:mm:ss} %-5p %m%n</Pattern>
+                        </PatternLayout>
+                        <Policies>
+                            <SizeBasedTriggeringPolicy size="100MB" />
+                        </Policies>
+                        <DefaultRolloverStrategy max="3" />
+                    </RollingFile>
+                </Route>
+            </Routes>
+            <IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
+        </Routing>
     </Appenders>
 
     <Loggers>
         <Root level="DEBUG">
             <AppenderRef ref="Default"/>
             <AppenderRef ref="LogSplit"/>
+            <AppenderRef ref="AppLogSplit"/>
         </Root>
 
         <Logger name="org.eclipse.jetty" level="INFO" />

diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java
@@ -121,4 +121,41 @@ void testLogNoJobRoot() throws InterruptedException {
     assertFalse(Files.exists(root.resolve(filename)));
   }
 
+  @Test
+  void testAppDispatch() throws InterruptedException {
+    final Logger logger = LoggerFactory.getLogger("testAppDispatch");
+
+    final String filename = "logs.log";
+
+    ExecutorService executor = Executors.newFixedThreadPool(1);
+    executor.submit(() -> {
+      MDC.put("workspace_app_root", root.toString());
+      logger.error("random message testAppDispatch");
+      MDC.clear();
+    });
+
+    executor.shutdown();
+    executor.awaitTermination(10, TimeUnit.SECONDS);
+
+    assertTrue(IOs.readFile(root, filename).contains("random message testAppDispatch"));
+  }
+
+  @Test
+  void testLogNoAppRoot() throws InterruptedException {
+    final Logger logger = LoggerFactory.getLogger("testAppDispatch");
+
+    final String filename = "logs.log";
+
+    ExecutorService executor = Executors.newFixedThreadPool(1);
+    executor.submit(() -> {
+      logger.error("random message testLogNoAppRoot");
+      MDC.clear();
+    });
+
+    executor.shutdown();
+    executor.awaitTermination(10, TimeUnit.SECONDS);
+
+    assertFalse(Files.exists(root.resolve(filename)));
+  }
+
 }
diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/SchedulerApp.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/SchedulerApp.java
@@ -49,6 +49,7 @@
 import java.util.concurrent.TimeUnit;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 
 /**
  * The SchedulerApp is responsible for finding new scheduled jobs that need to be run and to launch
@@ -115,11 +116,14 @@ private static ProcessBuilderFactory getProcessBuilderFactory(Configs configs) {
   }
 
   public static void main(String[] args) {
+
     final Configs configs = new EnvConfigs();
 
     final Path configRoot = configs.getConfigRoot();
     LOGGER.info("configRoot = " + configRoot);
 
+    MDC.put("workspace_app_root", configs.getWorkspaceRoot().resolve("scheduler/logs").toString());
+
     final Path workspaceRoot = configs.getWorkspaceRoot();
     LOGGER.info("workspaceRoot = " + workspaceRoot);
 

diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java
@@ -60,6 +60,7 @@
 import org.glassfish.jersey.servlet.ServletContainer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 
 public class ServerApp {
 
@@ -155,6 +156,8 @@ private static void setCustomerIdIfNotSet(final ConfigRepository configRepositor
   public static void main(String[] args) throws Exception {
     final Configs configs = new EnvConfigs();
 
+    MDC.put("workspace_app_root", configs.getWorkspaceRoot().resolve("server/logs").toString());
+
     final Path configRoot = configs.getConfigRoot();
     LOGGER.info("configRoot = " + configRoot);
 

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -64,6 +64,7 @@ services:
       - TRACKING_STRATEGY=${TRACKING_STRATEGY}
       - AIRBYTE_VERSION=${VERSION}
       - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
+      - WORKSPACE_ROOT=${WORKSPACE_ROOT}
     ports:
       - 8001:8001
     volumes:

diff --git a/kube/resources/server.yaml b/kube/resources/server.yaml
@@ -74,6 +74,11 @@ spec:
               value: airbyte-db-svc:5432
             - name: WAIT_HOSTS_TIMEOUT
               value: "45"
+            - name: WORKSPACE_ROOT
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: WORKSPACE_ROOT
           ports:
             - containerPort: 8001
           volumeMounts: