add arg chunk_size for all insert_rows() (#669)

elementary-data · Feb 25, 2024 · f58bda2 · f58bda2
1 parent ced5c8a
commit f58bda2
Show file tree

Hide file tree

Showing 5 changed files with 6 additions and 6 deletions.
diff --git a/macros/edr/dbt_artifacts/upload_dbt_invocation.sql b/macros/edr/dbt_artifacts/upload_dbt_invocation.sql
@@ -45,7 +45,7 @@
       'account_id': elementary.get_var("account_id", ["DBT_ACCOUNT_ID"]),
       'target_adapter_specific_fields': elementary.get_target_adapter_specific_fields()
   } %}
-  {% do elementary.insert_rows(relation, [dbt_invocation], should_commit=true) %}
+  {% do elementary.insert_rows(relation, [dbt_invocation], should_commit=true, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
   {% do elementary.file_log("Uploaded dbt invocation successfully.") %}
 {% endmacro %}
 

diff --git a/macros/edr/tests/on_run_end/handle_tests_results.sql b/macros/edr/tests/on_run_end/handle_tests_results.sql
@@ -15,11 +15,11 @@
     {% do elementary.insert_schema_columns_snapshot(database_name, schema_name, test_columns_snapshot_tables) %}
     {% if test_result_rows %}
       {% set test_result_rows_relation = elementary.get_elementary_relation('test_result_rows') %}
-      {% do elementary.insert_rows(test_result_rows_relation, test_result_rows, should_commit=True) %}
+      {% do elementary.insert_rows(test_result_rows_relation, test_result_rows, should_commit=True, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
     {% endif %}
     {% if elementary_test_results %}
       {% set elementary_test_results_relation = elementary.get_elementary_relation('elementary_test_results') %}
-      {% do elementary.insert_rows(elementary_test_results_relation, elementary_test_results, should_commit=True) %}
+      {% do elementary.insert_rows(elementary_test_results_relation, elementary_test_results, should_commit=True, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
     {% endif %}
     {% do elementary.file_log("Handled test results successfully.") %}
     {% do return('') %}

diff --git a/macros/edr/tests/on_run_end/insert_metrics.sql b/macros/edr/tests/on_run_end/insert_metrics.sql
@@ -7,5 +7,5 @@
   {% endif %}
 
   {{ elementary.file_log("Inserting {} metrics into {}.".format(metrics | length, target_relation)) }}
-  {% do elementary.insert_rows(target_relation, metrics, should_commit=true) %}
+  {% do elementary.insert_rows(target_relation, metrics, should_commit=true, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
 {% endmacro %}
diff --git a/macros/edr/tests/test_utils/backfill_result_rows.sql b/macros/edr/tests/test_utils/backfill_result_rows.sql
@@ -31,6 +31,6 @@
             }) %}
         {% endfor %}
     {% endfor %}
-    {% do elementary.insert_rows(this, test_result_rows_rows) %}
+    {% do elementary.insert_rows(this, test_result_rows_rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
     {% do return('') %}
 {% endmacro %}
diff --git a/macros/edr/tests/test_utils/create_model_baseline_table.sql b/macros/edr/tests/test_utils/create_model_baseline_table.sql
@@ -3,6 +3,6 @@
     {% set baseline_table_relation = elementary.create_elementary_test_table(database_name, schema_name,
                                                                              test_name | lower, 'schema_baseline',
                                                                              empty_table_query) %}
-    {% do elementary.insert_rows(baseline_table_relation, baseline_columns, should_commit=True) %}
+    {% do elementary.insert_rows(baseline_table_relation, baseline_columns, should_commit=True, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
     {% do return(baseline_table_relation) %}
 {% endmacro %}