Introducing SAAS Sources in OSI

.gitignore

@@ -16,3 +16,6 @@ data-prepper-main/src/test/resources/logstash-conf/logstash-filter.yaml
 
 # Python virtual environments
 .venv
+
+# Grale output folder
+**/out/

data-prepper-api/build.gradle

@@ -18,6 +18,11 @@ dependencies {
     testImplementation project(':data-prepper-test-common')
     testImplementation 'org.skyscreamer:jsonassert:1.5.3'
     testImplementation libs.commons.io
+
+    implementation(libs.spring.context) {
+        exclude group: 'commons-logging', module: 'commons-logging'
+    }
+
 }
 
 jacocoTestCoverageVerification {

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/DataPrepperPlugin.java

@@ -72,4 +72,13 @@
      * @since 1.2
      */
     Class<?> pluginConfigurationType() default PluginSetting.class;
+
+    /**
+     * This is an optional argument.
+     * When provided, it indicates that this plugin want to use
+     * Dependency Injection in its module, with the Spring ApplicationContext isolated to its own module.
+     *
+     * @return an Array of base packages of this plugin to scan and create Spring DI context.
+     */
+    Class[] packagesToScanForDI() default {};
 }

data-prepper-core/src/main/java/org/opensearch/dataprepper/parser/PipelineTransformer.java

@@ -128,6 +128,7 @@ private void buildPipelineFromConfiguration(
                 try {
                     return pluginFactory.loadPlugin(Source.class, sourceSetting);
                 } catch (Exception e) {
+                    LOG.error("Failed to instantiate the plugin class", e);
                     final PluginError pluginError = PluginError.builder()
                             .componentType(PipelineModel.SOURCE_PLUGIN_TYPE)
                             .pipelineName(pipelineName)

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java

@@ -13,11 +13,13 @@
 import org.opensearch.dataprepper.model.sink.SinkContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.BeanFactory;
 import org.springframework.context.annotation.DependsOn;
 
 import javax.inject.Inject;
 import javax.inject.Named;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
@@ -115,13 +117,16 @@ private <T> ComponentPluginArgumentsContext getConstructionContext(final PluginS
         final PluginConfigObservable pluginConfigObservable = pluginConfigurationObservableFactory
                 .createDefaultPluginConfigObservable(pluginConfigurationConverter, pluginConfigurationType, pluginSetting);
 
+        BeanFactory beanFactory =
+                pluginBeanFactoryProvider.initializePluginSpecificIsolatedContextCombinedWithShared(pluginAnnotation);
+
         return new ComponentPluginArgumentsContext.Builder()
                 .withPluginSetting(pluginSetting)
                 .withPipelineDescription(pluginSetting)
                 .withPluginConfiguration(configuration)
                 .withPluginFactory(this)
                 .withSinkContext(sinkContext)
-                .withBeanFactory(pluginBeanFactoryProvider.get())
+                .withBeanFactory(beanFactory)
                 .withPluginConfigurationObservable(pluginConfigObservable)
                 .withTypeArgumentSuppliers(applicationContextToTypedSuppliers.getArgumentsSuppliers())
                 .build();

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java

@@ -5,13 +5,16 @@
 
 package org.opensearch.dataprepper.plugin;
 
+import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
 import org.springframework.beans.factory.BeanFactory;
 import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 import org.springframework.context.support.GenericApplicationContext;
 
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Provider;
+import java.util.Arrays;
 import java.util.Objects;
 
 /**
@@ -25,7 +28,7 @@
  * <p><i>publicContext</i> is the root {@link ApplicationContext}</p>
  */
 @Named
-class PluginBeanFactoryProvider implements Provider<BeanFactory> {
+class PluginBeanFactoryProvider {
     private final GenericApplicationContext sharedPluginApplicationContext;
     private final GenericApplicationContext coreApplicationContext;
 
@@ -57,8 +60,17 @@ GenericApplicationContext getCoreApplicationContext() {
      * instead, a new isolated {@link ApplicationContext} should be created.
      * @return BeanFactory A BeanFactory that inherits from {@link PluginBeanFactoryProvider#sharedPluginApplicationContext}
      */
-    public BeanFactory get() {
-        final GenericApplicationContext isolatedPluginApplicationContext = new GenericApplicationContext(sharedPluginApplicationContext);
-        return isolatedPluginApplicationContext.getBeanFactory();
+    public BeanFactory initializePluginSpecificIsolatedContextCombinedWithShared(DataPrepperPlugin pluginAnnotation) {
+        AnnotationConfigApplicationContext pluginDIContext = new AnnotationConfigApplicationContext();
+        if(pluginAnnotation.packagesToScanForDI().length>0) {
+            // If packages to scan is provided in this plugin annotation, which indicates
+            // that this plugin is interested in using Dependency Injection isolated for its module
+            Arrays.stream(pluginAnnotation.packagesToScanForDI())
+                    .map(Class::getPackageName)
+                    .forEach(pluginDIContext::scan);
+            pluginDIContext.refresh();
+        }
+        pluginDIContext.setParent(sharedPluginApplicationContext);
+        return pluginDIContext.getBeanFactory();
     }
 }

data-prepper-plugins/build.gradle

@@ -13,5 +13,6 @@ subprojects {
 }
 
 dependencies {
-    subprojects.forEach { api project(':data-prepper-plugins:' + it.name) }
+    subprojects.findAll { api project(it.path) }
 }
+

data-prepper-plugins/saas-source-plugins/build.gradle

@@ -0,0 +1,13 @@
+plugins {
+    id 'java-library'
+}
+
+
+subprojects {
+    apply plugin: 'data-prepper.publish'
+    group = 'org.opensearch.dataprepper.plugins.source'
+}
+
+dependencies {
+    subprojects.forEach { api project(':data-prepper-plugins::saas-source-plugins:' + it.name) }
+}

data-prepper-plugins/saas-source-plugins/jira-source/build.gradle

@@ -0,0 +1,32 @@
+plugins {
+    id 'java'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+
+    implementation project(path: ':data-prepper-plugins:saas-source-plugins:saas-crawler')
+    implementation project(path: ':data-prepper-api')
+    implementation project(path: ':data-prepper-plugins:aws-plugin-api')
+    implementation project(path: ':data-prepper-plugins:buffer-common')
+
+    implementation 'com.fasterxml.jackson.core:jackson-core'
+    implementation 'com.fasterxml.jackson.core:jackson-databind'
+
+    implementation 'org.projectlombok:lombok:1.18.30'
+    annotationProcessor 'org.projectlombok:lombok:1.18.30'
+
+    testImplementation platform('org.junit:junit-bom:5.10.0')
+    testImplementation 'org.junit.jupiter:junit-jupiter'
+
+    implementation(libs.spring.context) {
+        exclude group: 'commons-logging', module: 'commons-logging'
+    }
+}
+
+test {
+    useJUnitPlatform()
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/saas/jira/JiraItemInfo.java

@@ -0,0 +1,10 @@
+package org.opensearch.dataprepper.plugins.source.saas.jira;
+
+import org.opensearch.dataprepper.plugins.source.saas.crawler.base.ItemInfo;
+
+public class JiraItemInfo extends ItemInfo {
+    @Override
+    public String getKeyAttributes() {
+        return "project|issue-type";
+    }
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/saas/jira/JiraSource.java

@@ -0,0 +1,40 @@
+package org.opensearch.dataprepper.plugins.source.saas.jira;
+
+
+import org.opensearch.dataprepper.metrics.PluginMetrics;
+import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager;
+import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
+import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
+import org.opensearch.dataprepper.model.plugin.PluginFactory;
+import org.opensearch.dataprepper.model.source.Source;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.SaasCrawlerApplicationContextMarker;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.base.Crawler;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.base.SaasSourcePlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * JiraConnector connector entry point.
+ */
+
+@DataPrepperPlugin(name = "jira",
+        pluginType = Source.class,
+        pluginConfigurationType = JiraSourceConfig.class,
+        packagesToScanForDI = {SaasCrawlerApplicationContextMarker.class, JiraSource.class}
+)
+public class JiraSource extends SaasSourcePlugin {
+
+  private static final Logger log = LoggerFactory.getLogger(JiraSource.class);
+
+  @DataPrepperPluginConstructor
+  public JiraSource(final PluginMetrics pluginMetrics,
+                    final JiraSourceConfig jiraSourceConfig,
+                    final PluginFactory pluginFactory,
+                    final AcknowledgementSetManager acknowledgementSetManager,
+                    Crawler crawler) {
+    super(pluginMetrics, jiraSourceConfig, pluginFactory, acknowledgementSetManager, crawler);
+    log.info("Create Jira Source Connector");
+  }
+
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/saas/jira/JiraSourceConfig.java

@@ -0,0 +1,72 @@
+package org.opensearch.dataprepper.plugins.source.saas.jira;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.base.SaasSourceConfig;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@Getter
+public class JiraSourceConfig implements SaasSourceConfig {
+
+    /**
+     * This connector's account url
+     */
+    @JsonProperty("account_url")
+    private String accountUrl;
+
+    /**
+     * A map of connector credentials specific to this connector
+     */
+    @JsonProperty("connector_credentials")
+    private Map<String, Object> connectorCredentials;
+
+    /**
+     * List of projects to ingest
+     */
+    @JsonProperty("project")
+    private List<String> project = new ArrayList<>();
+
+    /**
+     * List of specific issue types to ingest.
+     * Ex: Story, Epic, Task etc
+     */
+    @JsonProperty("issue_type")
+    private List<String> issueType = new ArrayList<>();
+
+    /**
+     * Optional Inclusion patterns for filtering some tickets
+     */
+    @JsonProperty("inclusion_patterns")
+    private String inclusionPatterns;
+
+    /**
+     * Optional Exclusion patterns for excluding some tickets
+     */
+    @JsonProperty("exclusion_patterns")
+    private String exclusionPatterns;
+
+    /**
+     * Optional Status filter to ingest the tickets
+     */
+    @JsonProperty("status")
+    private String status;
+
+    /**
+     * Number of worker threads to spawn to parallel source fetching
+     */
+    @JsonProperty("workers")
+    private int numWorkers = DEFAULT_NUMBER_OF_WORKERS;
+
+    /**
+     * Default time to wait (with exponential backOff) in the case of
+     * waiting for the source service to respond
+     */
+    @JsonProperty("backoff_time")
+    private Duration backOff = DEFAULT_BACKOFF_MILLIS;
+
+
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/saas/jira/JiraSourceIterator.java

@@ -0,0 +1,26 @@
+package org.opensearch.dataprepper.plugins.source.saas.jira;
+
+import org.opensearch.dataprepper.plugins.source.saas.crawler.base.ItemInfo;
+import org.springframework.stereotype.Component;
+
+import java.util.Iterator;
+import java.util.UUID;
+
+@Component
+public class JiraSourceIterator implements Iterator<ItemInfo> {
+
+    int itemCounter = 0;
+
+    @Override
+    public boolean hasNext() {
+        return itemCounter<10;
+    }
+
+    @Override
+    public ItemInfo next() {
+        ItemInfo itemInfo = new JiraItemInfo();
+        itemInfo.setId(UUID.randomUUID().toString());
+        itemCounter++;
+        return itemInfo;
+    }
+}

data-prepper-plugins/saas-source-plugins/saas-crawler/build.gradle

@@ -0,0 +1,35 @@
+plugins {
+    id 'java-library'
+}
+
+group = 'org.opensearch.dataprepper.plugins.source.saas.crawler'
+
+tasks.withType(Javadoc).configureEach {
+    enabled = false
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+
+    implementation project(path: ':data-prepper-api')
+
+    implementation 'com.fasterxml.jackson.core:jackson-core'
+    implementation 'com.fasterxml.jackson.core:jackson-databind'
+    implementation 'org.projectlombok:lombok:1.18.30'
+
+    implementation(libs.spring.context) {
+        exclude group: 'commons-logging', module: 'commons-logging'
+    }
+
+    annotationProcessor 'org.projectlombok:lombok:1.18.30'
+
+    testImplementation platform('org.junit:junit-bom:5.10.0')
+    testImplementation 'org.junit.jupiter:junit-jupiter'
+}
+
+test {
+    useJUnitPlatform()
+}

data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas/crawler/SaasCrawlerApplicationContextMarker.java

@@ -0,0 +1,7 @@
+package org.opensearch.dataprepper.plugins.source.saas.crawler;
+
+/**
+ * Market interface to indicate the base package to scan for dependency injection
+ */
+public interface SaasCrawlerApplicationContextMarker {
+}