source.yml

version: 2
sources:
  - name: delta_source
    config:
      plugin: delta
      #materialization: view
    tables: 
    - name: customer
      meta:
        delta_table_path: "/home/aleks/git/my-projects/banchmark-utils/utils/sf1/delta/customer"
        #as_of_version: 2 # https://delta-io.github.io/delta-rs/python/usage.html#time-travel
        #as_of_datetime: "2021-11-04 00:05:23.283+00:00" #crazy datetime https://delta-io.github.io/delta-rs/python/api_reference.html#deltalake.table.DeltaTable.load_with_datetime
    - name: customer1
      meta:
        delta_table_path: "/home/aleks/git/open-source/dbt-duckdb-delta-plugin-demo/dbt_duckdb_delta/data/customer1"
    - name: lineitem
      meta:
        delta_table_path: "/home/aleks/git/my-projects/banchmark-utils/utils/sf1/delta/lineitem"
        #you can take a parameter from var https://docs.getdbt.com/docs/build/project-variables
    
    ##AZURE CONNECTION
    ##NOTE: to use environment variables you have to write it in upercase and it has to start with azure e.g AZURE_STORAGE_ACCOUNT_KEY=<key>
    ##because delta-rs filters ENV variables with *AZURE_*  
    ##following documentations has all possible keys https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html#variants
    ##but be aware that it has to start with *AZURE_*

    #If you provide storage_options directly it will overwrite per table env variable
    - name: orders
      meta:
        delta_table_path: "abfss://test@<storage-account-name>.dfs.core.windows.net/region"
        # storage_options: 
        #   azure_storage_account_key: "{{ env_var('AZURE_STORAGE_ACCOUNT_KEY') }}" #https://docs.getdbt.com/reference/dbt-jinja-functions/env_var

    ## You can also exchange parts of the path strings 
    ##dbt -d run -select orders_raw --vars '{"storage_name": "<storage_name>"}
    # - name: orders
    #   meta:
    #     delta_table_path: "abfss://test@{{ var('storage_name') }}.dfs.core.windows.net/region"
    #     storage_options: 
    #       azure_storage_account_key: "{{ env_var('AZURE_STORAGE_ACCOUNT_KEY_2') }}"
  - name: external_source
    meta:
      external_location: "read_csv_auto('{prefix}{file}')"
      prefix: "/home/aleks/git/my-projects/banchmark-utils/utils/sf1/csv/"
    tables:
      - name: customer
        meta:
          #prefix: "override"
          #external_location: "override/{prefix}{file}" # das würde auch funktionieren
          file: "customer.csv"