update retry

zilliztech · Oct 31, 2024 · bdb669d · bdb669d
1 parent f6e7a13
commit bdb669d
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/...n/java/org/apache/seatunnel/connectors/seatunnel/milvus/sink/MilvusBufferBatchWriter.java b/...n/java/org/apache/seatunnel/connectors/seatunnel/milvus/sink/MilvusBufferBatchWriter.java
@@ -280,7 +280,7 @@ private void upsertWrite(String partitionName, List<JsonObject> data)
         } catch (Exception e) {
             if (e.getMessage().contains("rate limit exceeded")
                     || e.getMessage().contains("received message larger than max")) {
-                if (data.size() > 10) {
+                if (data.size() > 2) {
                     log.warn("upsert data failed, retry in smaller chunks: {} ", data.size() / 2);
                     this.batchSize = this.batchSize / 2;
                     log.info("sleep 1 minute to avoid rate limit");
@@ -306,7 +306,6 @@ private void upsertWrite(String partitionName, List<JsonObject> data)
                         e);
             }
         }
-        log.info("upsert data success");
     }
 
     private void insertWrite(String partitionName, List<JsonObject> data) {

diff --git a/...main/java/org/apache/seatunnel/connectors/seatunnel/milvus/source/MilvusSourceReader.java b/...main/java/org/apache/seatunnel/connectors/seatunnel/milvus/source/MilvusSourceReader.java
@@ -273,7 +273,7 @@ private void queryIteratorData(
                 }
             }
         } catch (Exception e) {
-            if (e.getMessage().contains("rate limit exceeded") && batchSize > 10) {
+            if ((e.getMessage().contains("rate limit exceeded") || e.getMessage().contains("received message larger than max")) && batchSize > 1) {
                 log.error(
                         "Query Iterate data from milvus failed, retry from beginning with smaller batch size: {} after 30 s",
                         batchSize / 2,