copy seed configurations from config/init to server (#4417)

Co-authored-by: subodh <[email protected]>

airbyte-config/init/build.gradle

@@ -7,3 +7,42 @@ dependencies {
 
     implementation project(':airbyte-config:models')
 }
+
+// generate seed for each yaml file.
+task generateSeed {
+    def seeds = [
+            [
+                "sourceDefinitionId",
+                new File(project.projectDir, '/src/main/resources/seed/source_definitions.yaml'),
+                new File(project.projectDir, '/src/main/resources/config/STANDARD_SOURCE_DEFINITION')
+            ],
+            [
+                "destinationDefinitionId",
+                new File(project.projectDir, '/src/main/resources/seed/destination_definitions.yaml'),
+                new File(project.projectDir, '/src/main/resources/config/STANDARD_DESTINATION_DEFINITION')
+            ],
+    ]
+    seeds.each{val ->
+        def name = val[0]
+        def taskName = "generateSeed$name"
+        dependsOn taskName
+        task "$taskName"(type: JavaExec) {
+            classpath = sourceSets.main.runtimeClasspath
+
+            main = 'io.airbyte.config.init.SeedRepository'
+
+            // arguments to pass to the application
+            args '--id-name'
+            args val[0]
+            args '--input-path'
+            args val[1]
+            args '--output-path'
+            args val[2]
+        }
+    }
+}
+
+// we only want to attempt generateSeed if tests have passed.
+generateSeed.dependsOn(check)
+generateSeed.dependsOn(assemble)
+build.dependsOn(generateSeed)

airbyte-config/init/src/main/java/io/airbyte/config/init/SeedRepository.java

@@ -0,0 +1,107 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.config.init;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.commons.io.IOs;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.config.helpers.YamlListToStandardDefinitions;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Collectors;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+/**
+ * This class takes in a yaml file with a list of objects. It then then assigns each object a uuid
+ * based on its name attribute. The uuid is written as a field in the object with the key specified
+ * as the id-name. It then writes each object to its own file in the specified output directory.
+ * Each file's name is the generated uuid. The goal is that a user should be able to add objects to
+ * the database seed without having to generate uuids themselves. The output files should be
+ * compatible with our file system database (config persistence).
+ */
+public class SeedRepository {
+
+  private static final Options OPTIONS = new Options();
+  private static final Option ID_NAME_OPTION = new Option("id", "id-name", true, "field name of the id");
+  private static final Option INPUT_PATH_OPTION = new Option("i", "input-path", true, "path to input file");
+  private static final Option OUTPUT_PATH_OPTION = new Option("o", "output-path", true, "path to where files will be output");
+
+  static {
+    ID_NAME_OPTION.setRequired(true);
+    INPUT_PATH_OPTION.setRequired(true);
+    OUTPUT_PATH_OPTION.setRequired(true);
+    OPTIONS.addOption(ID_NAME_OPTION);
+    OPTIONS.addOption(INPUT_PATH_OPTION);
+    OPTIONS.addOption(OUTPUT_PATH_OPTION);
+  }
+
+  private static CommandLine parse(final String[] args) {
+    final CommandLineParser parser = new DefaultParser();
+    final HelpFormatter helpFormatter = new HelpFormatter();
+
+    try {
+      return parser.parse(OPTIONS, args);
+    } catch (final ParseException e) {
+      helpFormatter.printHelp("", OPTIONS);
+      throw new IllegalArgumentException(e);
+    }
+  }
+
+  public static void main(final String[] args) throws IOException {
+    final CommandLine parsed = parse(args);
+    final String idName = parsed.getOptionValue(ID_NAME_OPTION.getOpt());
+    final Path inputPath = Path.of(parsed.getOptionValue(INPUT_PATH_OPTION.getOpt()));
+    final Path outputPath = Path.of(parsed.getOptionValue(OUTPUT_PATH_OPTION.getOpt()));
+
+    new SeedRepository().run(idName, inputPath, outputPath);
+  }
+
+  public void run(final String idName, final Path input, final Path output) throws IOException {
+    final var jsonNode = YamlListToStandardDefinitions.verifyAndConvertToJsonNode(idName, IOs.readFile(input));
+    final var elementsIter = jsonNode.elements();
+
+    // clean output directory.
+    for (final Path file : Files.list(output).collect(Collectors.toList())) {
+      Files.delete(file);
+    }
+
+    // write to output directory.
+    while (elementsIter.hasNext()) {
+      final JsonNode element = Jsons.clone(elementsIter.next());
+      IOs.writeFile(
+          output,
+          element.get(idName).asText() + ".json",
+          Jsons.toPrettyString(element));
+    }
+  }
+
+}

airbyte-config/init/src/test/java/io/airbyte/config/init/SeedRepositoryTest.java

@@ -0,0 +1,127 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.config.init;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.ImmutableMap;
+import io.airbyte.commons.io.IOs;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.yaml.Yamls;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class SeedRepositoryTest {
+
+  private static final String CONFIG_ID = "configId";
+  private static final JsonNode OBJECT = Jsons.jsonNode(ImmutableMap.builder()
+      .put(CONFIG_ID, UUID.randomUUID())
+      .put("name", "barker")
+      .put("description", "playwright")
+      .build());
+
+  private Path input;
+  private Path output;
+
+  @BeforeEach
+  void setup() throws IOException {
+    input = Files.createTempDirectory("test_input").resolve("input.yaml");
+    output = Files.createTempDirectory("test_output");
+
+    writeSeedList(OBJECT);
+  }
+
+  @Test
+  void testWrite() throws IOException {
+    new SeedRepository().run(CONFIG_ID, input, output);
+    final JsonNode actual = Jsons.deserialize(IOs.readFile(output, OBJECT.get(CONFIG_ID).asText() + ".json"));
+    assertEquals(OBJECT, actual);
+  }
+
+  @Test
+  void testOverwrites() throws IOException {
+    new SeedRepository().run(CONFIG_ID, input, output);
+    final JsonNode actual = Jsons.deserialize(IOs.readFile(output, OBJECT.get(CONFIG_ID).asText() + ".json"));
+    assertEquals(OBJECT, actual);
+
+    final JsonNode clone = Jsons.clone(OBJECT);
+    ((ObjectNode) clone).put("description", "revolutionary");
+    writeSeedList(clone);
+
+    new SeedRepository().run(CONFIG_ID, input, output);
+    final JsonNode actualAfterOverwrite = Jsons.deserialize(IOs.readFile(output, OBJECT.get(CONFIG_ID).asText() + ".json"));
+    assertEquals(clone, actualAfterOverwrite);
+  }
+
+  @Test
+  void testFailsOnDuplicateId() {
+    final JsonNode object = Jsons.clone(OBJECT);
+    ((ObjectNode) object).put("name", "howard");
+
+    writeSeedList(OBJECT, object);
+    final SeedRepository seedRepository = new SeedRepository();
+    assertThrows(IllegalArgumentException.class, () -> seedRepository.run(CONFIG_ID, input, output));
+  }
+
+  @Test
+  void testFailsOnDuplicateName() {
+    final JsonNode object = Jsons.clone(OBJECT);
+    ((ObjectNode) object).put(CONFIG_ID, UUID.randomUUID().toString());
+
+    writeSeedList(OBJECT, object);
+    final SeedRepository seedRepository = new SeedRepository();
+    assertThrows(IllegalArgumentException.class, () -> seedRepository.run(CONFIG_ID, input, output));
+  }
+
+  @Test
+  void testPristineOutputDir() throws IOException {
+    IOs.writeFile(output, "blah.json", "{}");
+    assertEquals(1, Files.list(output).count());
+
+    new SeedRepository().run(CONFIG_ID, input, output);
+
+    // verify the file that the file that was already in the directory is gone.
+    assertEquals(1, Files.list(output).count());
+    assertEquals(OBJECT.get(CONFIG_ID).asText() + ".json", Files.list(output).collect(Collectors.toList()).get(0).getFileName().toString());
+  }
+
+  private void writeSeedList(JsonNode... seeds) {
+    final JsonNode seedList = Jsons.jsonNode(new ArrayList<>());
+    for (JsonNode seed : seeds) {
+      ((ArrayNode) seedList).add(seed);
+    }
+    IOs.writeFile(input, Yamls.serialize(seedList));
+  }
+
+}

airbyte-integrations/connector-templates/generator/plopfile.js

@@ -24,7 +24,7 @@ ${additionalMessage || ""}
 
 module.exports = function (plop) {
   const docRoot = '../../../docs/integrations';
-  const definitionRoot = '../../../airbyte-server/src/main/resources';
+  const definitionRoot = '../../../airbyte-config/init/src/main/resources';
 
   const pythonSourceInputRoot = '../source-python';
   const singerSourceInputRoot = '../source-singer';

airbyte-server/Dockerfile

@@ -14,7 +14,7 @@ RUN chmod +x wait
 COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN mkdir latest_seeds
-COPY build/resources/main/config latest_seeds
+COPY build/config_init/resources/main/config latest_seeds
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 

airbyte-server/build.gradle

@@ -41,6 +41,16 @@ dependencies {
     testImplementation "org.testcontainers:postgresql:1.15.1"
 }
 
+// we want to be able to access the generated db files from config/init when we build the server docker image.
+task copySeed(type: Copy, dependsOn: [project(':airbyte-config:init').processResources]) {
+    from "${project(':airbyte-config:init').buildDir}/resources/main/config"
+    into "${buildDir}/config_init/resources/main/config"
+}
+
+// need to make sure that the files are in the resource directory before copying.
+//project.tasks.copySeed.mustRunAfter(project(':airbyte-config:init').tasks.processResources)
+assemble.dependsOn(project.tasks.copySeed)
+
 application {
     mainClass = 'io.airbyte.server.ServerApp'
 }
@@ -60,44 +70,4 @@ run {
     environment "AIRBYTE_VERSION", env.VERSION
     environment "AIRBYTE_ROLE", System.getenv('AIRBYTE_ROLE')
     environment "TEMPORAL_HOST", "localhost:7233"
-
 }
-
-// generate seed for each yaml file.
-task generateSeed {
-    def seeds = [
-            [
-                    "sourceDefinitionId",
-                    new File(project.projectDir, '/src/main/resources/seed/source_definitions.yaml'),
-                    new File(project.projectDir, '/src/main/resources/config/STANDARD_SOURCE_DEFINITION')
-            ],
-            [
-                    "destinationDefinitionId",
-                    new File(project.projectDir, '/src/main/resources/seed/destination_definitions.yaml'),
-                    new File(project.projectDir, '/src/main/resources/config/STANDARD_DESTINATION_DEFINITION')
-            ],
-    ]
-    seeds.each{val ->
-        def name = val[0]
-        def taskName = "generateSeed$name"
-        dependsOn taskName
-        task "$taskName"(type: JavaExec) {
-            classpath = sourceSets.main.runtimeClasspath
-
-            main = 'io.airbyte.server.SeedRepository'
-
-            // arguments to pass to the application
-            args '--id-name'
-            args val[0]
-            args '--input-path'
-            args val[1]
-            args '--output-path'
-            args val[2]
-        }
-    }
-}
-
-// we only want to attempt generateSeed if tests have passed.
-generateSeed.dependsOn(check)
-generateSeed.dependsOn(assemble)
-build.dependsOn(generateSeed)

airbyte-server/seed.Dockerfile

@@ -4,4 +4,4 @@ WORKDIR /app
 
 # the sole purpose of this image is to seed the data volume with the default data
 # that the app should have when it is first installed.
-COPY build/resources/main/config seed/config
+COPY build/config_init/resources/main/config seed/config

airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java

@@ -43,9 +43,9 @@ public class AirbyteGithubStore {
 
   private static final String GITHUB_BASE_URL = "https://raw.githubusercontent.com";
   private static final String SOURCE_DEFINITION_LIST_LOCATION_PATH =
-      "/airbytehq/airbyte/master/airbyte-server/src/main/resources/seed/source_definitions.yaml";
+      "/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/seed/source_definitions.yaml";
   private static final String DESTINATION_DEFINITION_LIST_LOCATION_PATH =
-      "/airbytehq/airbyte/master/airbyte-server/src/main/resources/seed/destination_definitions.yaml";
+      "/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml";
 
   private static final HttpClient httpClient = HttpClient.newHttpClient();