🎉 Oracle source: emit state messages more frequently (airbytehq#16238)

* Activate intermediate state emission for oracle * Bump version * Use a different pat to avoid api rate limit * Revert "use a different pat to avoid api rate limit" This reverts commit aaf4502. * Use a different pat to avoid api rate limit This reverts commit d9d741e. * auto-bump connector version [ci skip] * Revert "Use a different pat to avoid api rate limit" This reverts commit bb3430b. Co-authored-by: Octavia Squidington III <[email protected]>
jhammarstedt · Oct 31, 2022 · e4c29bd · e4c29bd
1 parent b467f07
commit e4c29bd
Show file tree

Hide file tree

Showing 7 changed files with 15 additions and 7 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -686,7 +686,7 @@
 - name: Oracle DB
   sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83
   dockerRepository: airbyte/source-oracle
-  dockerImageTag: 0.3.20
+  dockerImageTag: 0.3.21
   documentationUrl: https://docs.airbyte.io/integrations/sources/oracle
   icon: oracle.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -6707,7 +6707,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-oracle:0.3.20"
+- dockerImage: "airbyte/source-oracle:0.3.21"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/oracle"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile
@@ -17,5 +17,5 @@ ENV TZ UTC
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.3.20
+LABEL io.airbyte.version=0.3.21
 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile
@@ -8,5 +8,5 @@ ENV TZ UTC
 COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.20
+LABEL io.airbyte.version=0.3.21
 LABEL io.airbyte.name=airbyte/source-oracle
diff --git a/...ctors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/...ctors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java
@@ -35,6 +35,7 @@ public class OracleSource extends AbstractJdbcSource<JDBCType> implements Source
   private static final Logger LOGGER = LoggerFactory.getLogger(OracleSource.class);
 
   public static final String DRIVER_CLASS = DatabaseDriver.ORACLE.getDriverClassName();
+  private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000;
 
   private List<String> schemas;
 
@@ -75,9 +76,9 @@ public JsonNode toDatabaseConfig(final JsonNode config) {
     final Protocol protocol = config.has(JdbcUtils.ENCRYPTION_KEY)
         ? obtainConnectionProtocol(config.get(JdbcUtils.ENCRYPTION_KEY), additionalParameters)
         : Protocol.TCP;
-    String connectionString;
+    final String connectionString;
     if (config.has(CONNECTION_DATA)) {
-      JsonNode connectionData = config.get(CONNECTION_DATA);
+      final JsonNode connectionData = config.get(CONNECTION_DATA);
       final String connectionType = connectionData.has("connection_type") ? connectionData.get("connection_type").asText()
           : UNRECOGNIZED;
       connectionString = switch (connectionType) {
@@ -197,14 +198,19 @@ protected String getJdbcParameterDelimiter() {
     return ";";
   }
 
+  @Override
+  protected int getStateEmissionFrequency() {
+    return INTERMEDIATE_STATE_EMISSION_FREQUENCY;
+  }
+
   public static void main(final String[] args) throws Exception {
     final Source source = OracleSource.sshWrappedSource();
     LOGGER.info("starting source: {}", OracleSource.class);
     new IntegrationRunner(source).run(args);
     LOGGER.info("completed source: {}", OracleSource.class);
   }
 
-  private String buildConnectionString(JsonNode config, String protocol, String connectionTypeName, String connectionTypeValue) {
+  private String buildConnectionString(final JsonNode config, final String protocol, final String connectionTypeName, final String connectionTypeValue) {
     return String.format(
         "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(%s=%s)))",
         protocol,

diff --git a/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java
@@ -473,6 +473,7 @@ protected AirbyteStateType getSupportedStateType(final JsonNode config) {
     return PostgresUtils.isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM;
   }
 
+  @Override
   protected int getStateEmissionFrequency() {
     return INTERMEDIATE_STATE_EMISSION_FREQUENCY;
   }

diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md
@@ -132,6 +132,7 @@ Airbyte has the ability to connect to the Oracle source with 3 network connectiv
 
 | Version | Date       | Pull Request | Subject                                         |
 |:--------|:-----------| :--- |:------------------------------------------------|
+| 0.3.21  | 2022-09-01 | [](https://github.com/airbytehq/airbyte/pull/) | Emit state messages more frequently |
 | 0.3.20  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
 | 0.3.19  | 2022-08-03 | [14953](https://github.com/airbytehq/airbyte/pull/14953) | Use Service Name to connect to database |
 | 0.3.18  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |