feature: create base tpdm/edfi_3 models iteration 2

models/staging/edfi_3/base/_edfi_3__base.yml

@@ -82,6 +82,9 @@ models:
   - name: base_ef3__programs
     config:
       tags: ['core']
+  - name: base_ef3__school_year_types
+    config:
+      tags: ['core']
   - name: base_ef3__schools
     config:
       tags: ['core']
@@ -132,6 +135,9 @@ models:
   - name: base_ef3__student_education_organization_associations
     config:
       tags: ['core']
+  - name: base_ef3__student_education_organization_responsibility_associations
+    config:
+      tags: ['core']
   - name: base_ef3__student_homeless_program_associations
     config:
       tags: ['homeless']
@@ -169,6 +175,9 @@ models:
   - name: base_ef3__students
     config:
       tags: ['core']
+  - name: base_ef3__survey_course_associations
+    config:
+      tags: ['survey']
   - name: base_ef3__survey_question_responses
     config:
       tags: ['survey']

models/staging/edfi_3/base/base_ef3__school_year_types.sql

@@ -0,0 +1,21 @@
+with school_year_types as (
+    {{ source_edfi3('school_year_types') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        filename,
+        file_row_number,
+        is_deleted,
+
+        v:id::string                      as record_guid,
+        v:schoolYear::string              as school_year,
+        v:currentSchoolYear::string       as current_school_year,
+        v:schoolYearDescription::string   as school_year_description
+
+    from school_year_types
+)
+select * from renamed

models/staging/edfi_3/base/base_ef3__staff_absence_events.sql

@@ -0,0 +1,26 @@
+with staff_absence_events as (
+    {{ source_edfi3('staff_absence_events') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                                  as record_guid,
+        v:eventDate::date                             as event_date,
+        v:absenceEventReason::string                  as absence_event_reason,
+        v:hoursAbsent::integer                        as hours_absent,
+        v:staffReference:staffUniqueId::integer       as staff_unique_id,
+        -- descriptors
+        {{ extract_descriptor('v:absenceEventCategoryDescriptor::string') }} as absence_event_category,
+        -- references
+        v:staffReference as school_reference
+
+    from staff_absence_events
+)
+select * from renamed

models/staging/edfi_3/base/base_ef3__student_discipline_incident_behavior_associations.sql

@@ -19,12 +19,12 @@ renamed as (
         {{ extract_descriptor('v:behaviorDescriptor::string') }} as behavior_type,
         -- references
         v:disciplineIncidentReference as discipline_incident_reference,
-        v:studentReference           as student_reference,
+        v:studentReference            as student_reference,
         -- lists
         v:disciplineIncidentParticipationCodes as v_discipline_incident_participation_codes,
 
         -- edfi extensions
         v:_ext as v_ext
     from student_discipline_incident_behavior
 )
-select * from renamed
+select * from renamed

models/staging/edfi_3/base/base_ef3__student_discipline_incident_non_offender_associations.sql

@@ -24,4 +24,4 @@ renamed as (
         v:_ext as v_ext
     from student_discipline_incident_nonoffender
 )
-select * from renamed
+select * from renamed

models/staging/edfi_3/base/base_ef3__survey_course_associations.sql

@@ -0,0 +1,25 @@
+with survey_question_responses as (
+    {{ source_edfi3('survey_question_responses') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                                          as record_guid,
+        v:courseReference:courseCode::string                  as course_code,
+        v:courseReference:educationOrganizationId::integer    as education_organization_id,
+        v:surveyReference:namespace::string                   as namespace,
+        v:surveyReference:surveyIdentifier::string            as survey_identifier,
+        -- references
+        v:courseReference as program_reference,
+        v:surveyReference as survey_reference
+
+    from survey_question_responses
+)
+select * from renamed

models/staging/edfi_3/base/base_ef3__survey_question_responses.sql

@@ -18,7 +18,7 @@ renamed as (
         v:surveyResponseReference:surveyResponseIdentifier::string    as survey_response_id,
         v:comment::string                                             as comment,
         v:noResponse::boolean                                         as no_response,
-        --references
+        -- references
         v:surveyQuestionReference   as survey_question_reference,
         v:surveyResponseReference   as survey_response_reference,
         -- lists

models/staging/edfi_3/base/base_ef3__survey_questions.sql

@@ -19,7 +19,7 @@ renamed as (
         v:questionText::string                                        as question_text,
         -- descriptors
         {{ extract_descriptor('v:questionFormDescriptor::string') }} as question_form,
-        --references
+        -- references
         v:surveyReference          as survey_reference,
         v:surveySectionReference   as survey_section_reference,
         -- lists

models/staging/edfi_3/stage/_edfi_3__stage.yml

@@ -401,6 +401,10 @@ models:
     config:
       tags: ['core']
 
+  - name: stg_ef3__school_year_types
+    config:
+      tags: ['core']
+
   - name: stg_ef3__schools
     config:
       tags: ['core']
@@ -460,6 +464,10 @@ models:
     config:
       tags: ['core']
 
+  - name: stg_ef3__staff_absence_events
+    config:
+      tags: ['core']
+
   - name: stg_ef3__staff_education_organization_assignment_associations
     config:
       tags: ['core']

models/staging/edfi_3/stage/stg_ef3__school_year_types.sql

@@ -0,0 +1,28 @@
+with school_year_types as (
+    select * from {{ ref('base_ef3__school_year_types') }}
+    where not is_deleted
+),
+keyed as (
+    select
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'api_year',
+                'lower(school_year)',
+                'lower(current_school_year)'
+            ]
+        )}} as k_school_year_types,
+        api_year as school_year,
+        school_year_types.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from school_year_types
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_school_year_types',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)

models/staging/edfi_3/stage/stg_ef3__staff_absence_events.sql

@@ -0,0 +1,28 @@
+with staff_absence_events as (
+    select * from {{ ref('base_ef3__staff_absence_events') }}
+),
+keyed as (
+    select
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'lower(absence_event_category)',
+                'lower(survey_identifier)',
+                'lower(namespace)',
+                'lower(course_code)'
+            ]
+        ) }} as k_staff_absence_events,
+        staff_absence_events.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from staff_absence_events
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_staff_absence_events',
+            order_by='api_year desc, pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/edfi_3/stage/stg_ef3__survey_course_associations.sql

@@ -0,0 +1,29 @@
+with base_survey_course_associations as (
+    select * from {{ ref('base_ef3__survey_course_associations') }}
+    where not is_deleted
+),
+keyed as (
+    select
+        {{ gen_skey('k_survey') }},
+        {{ gen_skey('k_course') }},
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'lower(education_organization_id)',
+                'lower(event_date)'
+            ]
+        ) }} as k_survey_course_associations,
+        base_survey_course_associations.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from base_survey_course_associations
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_survey, k_course, entry_date, k_survey_course_associations',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/tpdm/base/_tpdm__base.yml

@@ -7,28 +7,22 @@ models:
   - name: base_tpdm__educator_preparation_programs
     config:
       tags: ['tpdm']
-  - name: base_tpdm__survey_response_person_target_associations
-    config:
-      tags: ['tpdm']
   - name: base_tpdm__candidate_educator_preparation_program_associations
     config:
       tags: ['tpdm']
   - name: base_tpdm__financial_aids
     config:
       tags: ['tpdm']
-  - name: base_tpdm__rubric_dimensions
-    config:
-      tags: ['tpdm']
   - name: base_tpdm__evaluations
     config:
       tags: ['tpdm']
-  - name: base_tpdm__evaluation_ratings
+  - name: base_tpdm__evaluation_elements
     config:
       tags: ['tpdm']
-  - name: base_tpdm__performance_evaluations
+  - name: base_tpdm__evaluation_element_ratings
     config:
       tags: ['tpdm']
-  - name: base_tpdm__performance_evaluation_ratings
+  - name: base_tpdm__evaluation_ratings
     config:
       tags: ['tpdm']
   - name: base_tpdm__evaluation_objectives
@@ -37,9 +31,15 @@ models:
   - name: base_tpdm__evaluation_objective_ratings
     config:
       tags: ['tpdm']
-  - name: base_tpdm__evaluation_elements
+  - name: base_tpdm__performance_evaluation_ratings
     config:
       tags: ['tpdm']
-  - name: base_tpdm__evaluation_element_ratings
+  - name: base_tpdm__performance_evaluations
+    config:
+      tags: ['tpdm']
+  - name: base_tpdm__rubric_dimensions
+    config:
+      tags: ['tpdm']
+  - name: base_tpdm__survey_response_person_target_associations
     config:
       tags: ['tpdm']