Add snowflake lake loader support

dbt_project.yml

@@ -115,6 +115,7 @@ vars:
 
     # Snowflake:
     snowplow__query_tag: snowplow_dbt
+    snowplow__snowflake_lakeloader: false
 
     # Databricks:
     # Depending on the use case it should either be the catalog (for Unity Catalog users from databricks connector 1.1.1 onwards) or the same value as your snowplow__atomic_schema (unless changed it should be 'atomic')

macros/field_extractions/get_app_error_event_fields.sql

@@ -84,15 +84,28 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_app_error_event_fields() %}
   {% if var('snowplow__enable_app_errors', false) %}
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programmingLanguage::VARCHAR() AS app_error__programming_language
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:className::VARCHAR() AS app_error__class_name
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exceptionName::VARCHAR() AS app_error__exception_name
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:isFatal::BOOLEAN AS app_error__is_fatal
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:lineNumber::INT AS app_error__line_number
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stackTrace::VARCHAR() AS app_error__stack_trace
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadId::INT AS app_error__thread_id
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadName::VARCHAR() AS app_error__thread_name
+    {% if var('snowplow__snowflake_lakeloader', false) %}
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programming_language::VARCHAR() AS app_error__programming_language
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:class_name::VARCHAR() AS app_error__class_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exception_name::VARCHAR() AS app_error__exception_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:is_fatal::BOOLEAN AS app_error__is_fatal
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:line_number::INT AS app_error__line_number
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stack_trace::VARCHAR() AS app_error__stack_trace
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:thread_id::INT AS app_error__thread_id
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:thread_name::VARCHAR() AS app_error__thread_name
+    {% else %}
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programmingLanguage::VARCHAR() AS app_error__programming_language
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:className::VARCHAR() AS app_error__class_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exceptionName::VARCHAR() AS app_error__exception_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:isFatal::BOOLEAN AS app_error__is_fatal
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:lineNumber::INT AS app_error__line_number
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stackTrace::VARCHAR() AS app_error__stack_trace
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadId::INT AS app_error__thread_id
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadName::VARCHAR() AS app_error__thread_name
+    {% endif %}
+
   {% else %}
     , cast(null as {{ dbt.type_string() }}) as app_error__message
     , cast(null as {{ dbt.type_string() }}) as app_error__programming_language

macros/field_extractions/get_browser_context_fields.sql

@@ -105,19 +105,35 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_browser_context_fields() %}
   {% if var('snowplow__enable_browser_context', false) %}
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentSize::varchar AS browser__document_size
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:colorDepth::int AS browser__color_depth
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:devicePixelRatio::float AS browser__device_pixel_ratio
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookiesEnabled::boolean AS browser__cookies_enabled
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browserLanguage::varchar AS browser__browser_language
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentLanguage::varchar AS browser__document_language
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:deviceMemory::int AS browser__device_memory
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardwareConcurrency::int AS browser__hardware_concurrency
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tabId::varchar AS browser__tab_id
+    {% if var('snowplow__snowflake_lakeloader', false) %}
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:document_size::varchar AS browser__document_size
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:color_depth::int AS browser__color_depth
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:device_pixel_ratio::float AS browser__device_pixel_ratio
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookies_enabled::boolean AS browser__cookies_enabled
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browser_language::varchar AS browser__browser_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:document_language::varchar AS browser__document_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:device_memory::int AS browser__device_memory
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardware_concurrency::int AS browser__hardware_concurrency
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tab_id::varchar AS browser__tab_id
+    {% else %}
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentSize::varchar AS browser__document_size
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:colorDepth::int AS browser__color_depth
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:devicePixelRatio::float AS browser__device_pixel_ratio
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookiesEnabled::boolean AS browser__cookies_enabled
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browserLanguage::varchar AS browser__browser_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentLanguage::varchar AS browser__document_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:deviceMemory::int AS browser__device_memory
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardwareConcurrency::int AS browser__hardware_concurrency
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tabId::varchar AS browser__tab_id
+    {% endif %}
   {% else %}
     , cast(null as {{ dbt.type_string() }}) as browser__viewport
     , cast(null as {{ dbt.type_string() }}) as browser__document_size

macros/field_extractions/get_cmp_visible_event_fields.sql

@@ -46,7 +46,11 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_cmp_visible_event_fields() %}
     {% if var('snowplow__enable_consent', false) %}
-    , unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsedTime::float as cmp__elapsed_time
+      {% if var('snowplow__snowflake_lakeloader', false) %}
+        , unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsed_time::float as cmp__elapsed_time
+      {% else %}
+        , unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsedTime::float as cmp__elapsed_time
+      {% endif %}
     {% else %}
       , cast(null as {{ dbt.type_float() }}) as cmp__elapsed_time
     {% endif %}

macros/field_extractions/get_consent_event_fields.sql

@@ -74,23 +74,32 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 {% endmacro %}
 
 {% macro snowflake__get_consent_event_fields() %}
-    {% if var('snowplow__enable_consent', false) %}
-
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:eventType::varchar as consent__event_type
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basisForProcessing::varchar as consent__basis_for_processing
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentUrl::varchar as consent__consent_url
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentVersion::varchar as consent__consent_version
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentScopes::array as consent__consent_scopes
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domainsApplied::array as consent__domains_applied
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdprApplies::boolean as consent__gdpr_applies
-
+  {% if var('snowplow__enable_consent', false) %}
+    {% if var('snowplow__snowflake_lakeloader', false) %}
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:event_type::varchar as consent__event_type
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basis_for_processing::varchar as consent__basis_for_processing
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_url::varchar as consent__consent_url
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_version::varchar as consent__consent_version
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_scopes::array as consent__consent_scopes
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domains_applied::array as consent__domains_applied
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdpr_applies::boolean as consent__gdpr_applies
     {% else %}
-      , cast(null as {{ dbt.type_string() }}) as consent__event_type
-      , cast(null as {{ dbt.type_string() }}) as consent__basis_for_processing
-      , cast(null as {{ dbt.type_string() }}) as consent__consent_url
-      , cast(null as {{ dbt.type_string() }}) as consent__consent_version
-      , cast(null as array) as consent__consent_scopes
-      , cast(null as array) as consent__domains_applied
-      , cast(null as {{ dbt.type_boolean() }}) as consent__gdpr_applies
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:eventType::varchar as consent__event_type
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basisForProcessing::varchar as consent__basis_for_processing
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentUrl::varchar as consent__consent_url
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentVersion::varchar as consent__consent_version
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentScopes::array as consent__consent_scopes
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domainsApplied::array as consent__domains_applied
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdprApplies::boolean as consent__gdpr_applies
     {% endif %}
+
+  {% else %}
+    , cast(null as {{ dbt.type_string() }}) as consent__event_type
+    , cast(null as {{ dbt.type_string() }}) as consent__basis_for_processing
+    , cast(null as {{ dbt.type_string() }}) as consent__consent_url
+    , cast(null as {{ dbt.type_string() }}) as consent__consent_version
+    , cast(null as array) as consent__consent_scopes
+    , cast(null as array) as consent__domains_applied
+    , cast(null as {{ dbt.type_boolean() }}) as consent__gdpr_applies
+  {% endif %}
 {% endmacro %}

macros/field_extractions/get_cwv_fields.sql

@@ -74,13 +74,23 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_cwv_fields() %}
     {% if var('snowplow__enable_cwv', false) %}
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigationType::varchar as cwv__navigation_type
+      {% if var('snowplow__snowflake_lakeloader', false) %}
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigation_type::varchar as cwv__navigation_type
+      {% else %}
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigationType::varchar as cwv__navigation_type
+      {% endif %}
     {% else %}
     , cast(null as decimal(14,4)) as cwv__lcp,
     , cast(null as decimal(14,4)) as cwv__fcp,

macros/field_extractions/get_geo_context_fields.sql

@@ -74,13 +74,23 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_geo_context_fields() %}
   {% if var('snowplow__enable_geolocation_context', false) %}
-    , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude::float AS geo__latitude
-    , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:longitude::float AS geo__longitude
-    , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitudeLongitudeAccuracy::float AS geo__latitude_longitude_accuracy
-    , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude::float AS geo__altitude
-    , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitudeAccuracy::float AS geo__altitude_accuracy
-    , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:bearing::float AS geo__bearing
-    , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:speed::float AS geo__speed
+    {% if var('snowplow__snowflake_lakeloader', false) %}
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude::float AS geo__latitude
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:longitude::float AS geo__longitude
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude_longitude_accuracy::float AS geo__latitude_longitude_accuracy
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude::float AS geo__altitude
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude_accuracy::float AS geo__altitude_accuracy
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:bearing::float AS geo__bearing
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:speed::float AS geo__speed
+    {% else %}
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude::float AS geo__latitude
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:longitude::float AS geo__longitude
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitudeLongitudeAccuracy::float AS geo__latitude_longitude_accuracy
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude::float AS geo__altitude
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitudeAccuracy::float AS geo__altitude_accuracy
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:bearing::float AS geo__bearing
+      , contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:speed::float AS geo__speed
+    {% endif %}
   {% else %}
     , cast(null as {{ dbt.type_float() }}) as geo__latitude
     , cast(null as {{ dbt.type_float() }}) as geo__longitude