diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 382fe05..cf5e771 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -201,6 +201,25 @@ 'col_list': ['candidateIdentifier'], 'annualize': True }, + + 'k_applicant_profile': { + 'reference_name': 'applicant_profile_reference', + 'col_list': ['applicantProfileIdentifier'], + 'annualize': True + }, + + 'k_certification_exam': { + 'reference_name': 'certification_exam_reference', + 'col_list': ['certificationExamIdentifier', 'namespace'], + 'annualize': True + }, + + 'k_survey_section': { + 'reference_name': 'certification_exam_reference', + 'col_list': ['surveyIdentifier', 'namespace', 'survey_section_title'], + 'annualize': True + }, + 'k_person': { 'reference_name': 'person_reference', 'col_list': ['personId', 'sourceSystemDescriptor'], @@ -297,6 +316,7 @@ 'termDescriptor'], 'annualize': True }, + 'k_template': { 'reference_name': '', 'col_list': [], diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml index 50f4ff4..d92453e 100644 --- a/models/staging/edfi_3/base/_edfi_3__base.yml +++ b/models/staging/edfi_3/base/_edfi_3__base.yml @@ -197,3 +197,9 @@ models: - name: base_ef3__survey_section_responses config: tags: ['survey'] + - name: base_ef3__people + config: + tags: ['core'] + - name: base_ef3__school_year_types + config: + tags: ['core'] diff --git a/models/staging/edfi_3/base/base_ef3__credentials.sql b/models/staging/edfi_3/base/base_ef3__credentials.sql index 51816a1..9257cbe 100644 --- a/models/staging/edfi_3/base/base_ef3__credentials.sql +++ b/models/staging/edfi_3/base/base_ef3__credentials.sql @@ -11,16 +11,18 @@ renamed as ( filename, is_deleted, - v:id::string as record_guid, - v:credentialIdentifier::string as credential_id, - v:effectiveDate::date as effective_date, - v:expirationDate::date as expiration_date, - v:issuanceDate::date as issuance_date, - v:namespace::string as namespace, + v:id::string as record_guid, + -- identity components + v:credentialIdentifier::string as credential_id, + {{ extract_descriptor('v:stateOfIssueStateAbbreviationDescriptor::string') }} as state_of_issue_state_abbreviation, + -- non-identity components + v:effectiveDate::date as effective_date, + v:expirationDate::date as expiration_date, + v:issuanceDate::date as issuance_date, + v:namespace::string as namespace, -- descriptors {{ extract_descriptor('v:credentialFieldDescriptor::string') }} as credential_field, {{ extract_descriptor('v:credentialTypeDescriptor::string') }} as credential_type, - {{ extract_descriptor('v:stateOfIssueStateAbbreviationDescriptor::string') }} as state_of_issue_state_abbreviation, {{ extract_descriptor('v:teachingCredentialDescriptor::string') }} as teaching_credential, {{ extract_descriptor('v:teachingCredentialBasisDescriptor::string') }} as teaching_credential_basis, -- unnested lists diff --git a/models/staging/edfi_3/base/base_ef3__people.sql b/models/staging/edfi_3/base/base_ef3__people.sql new file mode 100644 index 0000000..6ad734c --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__people.sql @@ -0,0 +1,20 @@ +with people as ( + {{ source_edfi3('people') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:personId::int as person_id, + {{ extract_descriptor('v:sourceSystemDescriptor::string') }} as source_system + from people +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__post_secondary_institutions.sql b/models/staging/edfi_3/base/base_ef3__post_secondary_institutions.sql index 9ff872b..48ada05 100644 --- a/models/staging/edfi_3/base/base_ef3__post_secondary_institutions.sql +++ b/models/staging/edfi_3/base/base_ef3__post_secondary_institutions.sql @@ -11,8 +11,10 @@ renamed as ( filename, is_deleted, - v:id::string as record_guid, + v:id::string as record_guid, + -- identity components v:postSecondaryInstitutionId::int as post_secondary_institution_id, + -- non-identity components v:nameOfInstitution::string as name_of_institution, v:shortNameOfInstitution::string as short_name_of_institution, v:webSite::string as web_site, diff --git a/models/staging/edfi_3/base/base_ef3__school_year_types.sql b/models/staging/edfi_3/base/base_ef3__school_year_types.sql new file mode 100644 index 0000000..8e8c985 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__school_year_types.sql @@ -0,0 +1,22 @@ +with school_year_types as ( + {{ source_edfi3('school_year_types') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:schoolYear::string as school_year, + -- non-identity components + v:currentSchoolYear::string as current_school_year, + v:schoolYearDescription::string as school_year_description + from school_year_types +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_section_responses.sql b/models/staging/edfi_3/base/base_ef3__survey_section_responses.sql index 2746b03..488a3c6 100644 --- a/models/staging/edfi_3/base/base_ef3__survey_section_responses.sql +++ b/models/staging/edfi_3/base/base_ef3__survey_section_responses.sql @@ -11,10 +11,14 @@ renamed as ( filename, is_deleted, - v:id::string as record_guid, - v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, + v:id::string as record_guid, + -- identity components + v:surveyResponseReference:namespace::string as namespace, v:surveySectionReference:surveyIdentifier::string as survey_id, - v:sectionRating::float as section_rating, + v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, + v:surveySectionReference:surveySectionTitle::string as survey_section_title, + -- non-identity components + v:sectionRating::float as section_rating, -- references v:surveyResponseReference as survey_response_reference, v:surveySectionReference as survey_section_reference diff --git a/models/staging/edfi_3/base/base_ef3__survey_sections.sql b/models/staging/edfi_3/base/base_ef3__survey_sections.sql index 26dbbf5..54ca6b8 100644 --- a/models/staging/edfi_3/base/base_ef3__survey_sections.sql +++ b/models/staging/edfi_3/base/base_ef3__survey_sections.sql @@ -11,8 +11,10 @@ renamed as ( filename, is_deleted, - v:id::string as record_guid, - v:surveyReference:surveyIdentifier::string as survey_id, + v:id::string as record_guid, + -- identity components + v:surveyReference::surveyIdentifier::string as survey_id, + v:surveyReference::namespace::string as namespace, v:surveySectionTitle::string as survey_section_title, -- references v:surveyReference as survey_reference diff --git a/models/staging/edfi_3/stage/_edfi_3__stage.yml b/models/staging/edfi_3/stage/_edfi_3__stage.yml index 046b809..dc9fcff 100644 --- a/models/staging/edfi_3/stage/_edfi_3__stage.yml +++ b/models/staging/edfi_3/stage/_edfi_3__stage.yml @@ -127,6 +127,11 @@ referential_integrity_tests: field: k_survey_response tags: ['ref_integrity'] + - k_survey_section: &ref_k_survey_section + - relationships: + to: ref('stg_ef3__survey_sections') + field: k_survey_section + - k_person: &ref_k_person - relationships: to: ref('stg_ef3__people') @@ -853,3 +858,43 @@ models: config: tags: ['survey'] enabled: "{{ var('src:domain:survey:enabled', True) }}" + + - name: stg_ef3__people + config: + tags: ['core'] + + - name: stg_ef3__credentials + config: + tags: ['core'] + columns: + - name: k_student_academic_record + tests: + *ref_k_student_academic_record + + - name: stg_ef3__post_secondary_institutions + config: + tags: ['core'] + + - name: stg_ef3__survey_sections + config: + tags: ['core'] + columns: + - name: k_survey + tests: + *ref_k_survey + + - name: stg_ef3__survey_section_responses + config: + tags: ['core'] + columns: + - name: k_survey_response + tests: + *ref_k_survey_response + - name: k_survey_section + tests: + *ref_k_survey_section + + - name: stg_ef3__school_year_types + config: + tags: ['core'] + diff --git a/models/staging/edfi_3/stage/stg_ef3__credentials.sql b/models/staging/edfi_3/stage/stg_ef3__credentials.sql new file mode 100644 index 0000000..5ca515d --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__credentials.sql @@ -0,0 +1,29 @@ +with credentials as ( + select * from {{ ref('base_tpdm__credentials') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'lower(namespace)', + 'lower(person_id)', + 'lower(source_system)', + 'lower(survey_id)', + 'lower(survey_response_id)'] + ) }} as k_credential, + {{ gen_skey('k_student_academic_record')}}, + credentials.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from credentials +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_credential', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/edfi_3/stage/stg_ef3__people.sql b/models/staging/edfi_3/stage/stg_ef3__people.sql new file mode 100644 index 0000000..b1c3405 --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__people.sql @@ -0,0 +1,25 @@ +with base_people as ( + select * from {{ ref('base_ef3__people') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'person_id', + 'lower(source_system)'] + ) }} as k_person, + base_people.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_people +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_person', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/edfi_3/stage/stg_ef3__post_secondary_institutions.sql b/models/staging/edfi_3/stage/stg_ef3__post_secondary_institutions.sql new file mode 100644 index 0000000..611cb75 --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__post_secondary_institutions.sql @@ -0,0 +1,24 @@ +with post_secondary_institutions as ( + select * from {{ ref('base_tpdm__post_secondary_institutions') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'post_secondary_institution_id'] + ) }} as k_post_secondary_institution, + post_secondary_institutions.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from post_secondary_institutions +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_post_secondary_institution', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql b/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql new file mode 100644 index 0000000..83c61c5 --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql @@ -0,0 +1,24 @@ +with school_year_types as ( + select * from {{ ref('base_ef3__school_year_types') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'lower(school_year)'] + ) }} as k_school_year_type, + school_year_types.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from school_year_types +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_school_year_type', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/edfi_3/stage/stg_ef3__survey_section_responses.sql b/models/staging/edfi_3/stage/stg_ef3__survey_section_responses.sql new file mode 100644 index 0000000..57cb590 --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__survey_section_responses.sql @@ -0,0 +1,28 @@ +with survey_section_responses as ( + select * from {{ ref('base_ef3__survey_section_responses') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'lower(survey_id)', + 'lower(namespace)', + 'lower(survey_section_title)'] + ) }} as k_survey_section_response, + {{ gen_skey('k_survey_response') }}, + {{ gen_skey('k_survey_section') }}, + survey_section_responses.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from survey_section_responses +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_survey_section_response', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/edfi_3/stage/stg_ef3__survey_sections.sql b/models/staging/edfi_3/stage/stg_ef3__survey_sections.sql new file mode 100644 index 0000000..df94e9b --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__survey_sections.sql @@ -0,0 +1,27 @@ +with survey_sections as ( + select * from {{ ref('base_ef3__survey_sections') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'lower(survey_id)', + 'lower(namespace)', + 'lower(survey_section_title)'] + ) }} as k_survey_section, + {{ gen_skey('k_survey') }}, + survey_sections.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from survey_sections +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_survey_section', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/base/_tpdm__base.yml b/models/staging/tpdm/base/_tpdm__base.yml index 684b62e..d798f3e 100644 --- a/models/staging/tpdm/base/_tpdm__base.yml +++ b/models/staging/tpdm/base/_tpdm__base.yml @@ -43,3 +43,25 @@ models: - name: base_tpdm__evaluation_element_ratings config: tags: ['tpdm'] + - name: base_tpdm__applicant_profiles + config: + tags: ['tpdm'] + - name: base_tpdm__applications + config: + tags: ['tpdm'] + - name: base_tpdm__candidate_relationship_to_staff_associations + config: + tags: ['tpdm'] + - name: base_tpdm__certification_exam_results + config: + tags: ['tpdm'] + - name: base_tpdm__certification_exams + config: + tags: ['tpdm'] + - name: base_tpdm__certifications + config: + tags: ['tpdm'] + - name: base_tpdm__fieldwork_experiences + config: + tags: ['tpdm'] + - name: base_tpdm__survey_response_education_organization_target_associations diff --git a/models/staging/tpdm/base/base_tpdm__applicant_profiles.sql b/models/staging/tpdm/base/base_tpdm__applicant_profiles.sql new file mode 100644 index 0000000..16c7b0b --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__applicant_profiles.sql @@ -0,0 +1,42 @@ +with applicant_profiles as ( + {{ source_edfi3('applicant_profiles') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:applicantProfileIdentifier::string as applicant_profile_id, + -- non-identity components + v:firstName::string as first_name, + v:lastSurname::string as last_name, + -- descriptors + {{ extract_descriptor('v:sexDescriptor::string') }} as sex, + -- unnested lists + v:addresses as v_addresses, + v:applicantCharacteristics as v_applicant_characteristics, + v:backgroundChecks as v_background_checks, + v:disabilities as v_disabilities, + v:educatorPreparationProgramNames as v_educator_preparation_program_names, + v:electronicMails as v_electronic_mails, + v:gradePointAverages as v_grade_point_averages, + v:highlyQualifiedAcademicSubjects as v_highly_qualified_academic_subjects, + v:identificationDocuments as v_identification_documents, + v:internationalAddresses as v_international_addresses, + v:languages as v_languages, + v:personalIdentificationDocuments as v_personal_identification_documents, + v:races as v_races, + v:telephones as v_telephones, + v:visas as v_visas, + -- edfi extensions + v:_ext as v_ext + from applicant_profiles +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__applications.sql b/models/staging/tpdm/base/base_tpdm__applications.sql new file mode 100644 index 0000000..1924332 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__applications.sql @@ -0,0 +1,36 @@ +with applications as ( + {{ source_edfi3('applications') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:applicantProfileReference:applicantProfileIdentifier::string as applicant_profile_id, + v:applicationIdentifier::string as application_id, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:educationOrganizationReference:link:rel::string as ed_org_type, + v:acceptedDate::date as accepted_date, + v:applicationDate::date as application_date, + -- descriptors + {{ extract_descriptor('v:applicationStatusDescriptor::string') }} as application_status, + -- references + v:applicantProfileReference as applicant_profile_reference, + v:educationOrganizationReference as education_organization_reference, + -- unnested lists + v:recruitmentEventAttendances as v_recruitment_event_attendances, + v:scoreResults as v_score_results, + v:terms as v_terms, + -- edfi extensions + v:_ext as v_ext + from applications +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__candidate_relationship_to_staff_associations.sql b/models/staging/tpdm/base/base_tpdm__candidate_relationship_to_staff_associations.sql new file mode 100644 index 0000000..1937fcc --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__candidate_relationship_to_staff_associations.sql @@ -0,0 +1,29 @@ +with candidate_relationship_to_staff_associations as ( + {{ source_edfi3('candidate_relationship_to_staff_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:candidateReference:candidateIdentifier::string as candidate_id, + v:staffReference:staffUniqueId::string as staff_unique_id, + -- non-identity components + v:beginDate::date as begin_date, + -- descriptors + {{ extract_descriptor('v:staffToCandidateRelationshipDescriptor::string') }} as staff_to_candidate_relationship, + -- references + v:candidateReference as candidate_reference, + v:staffReference as staff_reference, + -- edfi extensions + v:_ext as v_ext + from candidate_relationship_to_staff_associations +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__certification_exam_results.sql b/models/staging/tpdm/base/base_tpdm__certification_exam_results.sql new file mode 100644 index 0000000..8e6e5f9 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__certification_exam_results.sql @@ -0,0 +1,31 @@ +with certification_exam_results as ( + {{ source_edfi3('certification_exam_results') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:certificationExamDate::date as certification_exam_date, + v:certificationExamReference:certificationExamIdentifier::string as certification_exam_id, + v:certificationExamReference:namespace::string as namespace, + v:personReference:personId::string as person_id, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string')}} as source_system, + -- non-identity components + v:attemptNumber::int as attempt_number, + v:certificationExamPassIndicator::boolean as certification_exam_pass_indicator, + -- descriptors + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system_descriptor, + -- references + v:certificationExamReference as certification_exam_reference, + v:personReference as person_reference + from certification_exam_results +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__certification_exams.sql b/models/staging/tpdm/base/base_tpdm__certification_exams.sql new file mode 100644 index 0000000..de54be0 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__certification_exams.sql @@ -0,0 +1,26 @@ +with certification_exams as ( + {{ source_edfi3('certification_exams') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:certificationExamIdentifier::string as certification_exam_id, + v:namespace::string as namespace, + -- non-identity components + v:certificationExamTitle::string as certification_exam_title, + v:effectiveDate::date as effective_date, + v:endDate::date as end_date, + -- descriptors + {{ extract_descriptor('v:certificationExamTypeDescriptor::string') }} as certification_exam_type + from certification_exams +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__certifications.sql b/models/staging/tpdm/base/base_tpdm__certifications.sql new file mode 100644 index 0000000..25bbde4 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__certifications.sql @@ -0,0 +1,34 @@ +with certifications as ( + {{ source_edfi3('certifications') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:certificationIdentifier::string as certification_id, + v:namespace::string as namespace, + -- non-identity components + v:certificationTitle::string as certification_title, + -- descriptors + {{ extract_descriptor('v:certificationFieldDescriptor::string') }} as certification_field, + {{ extract_descriptor('v:certificationLevelDescriptor::string') }} as certification_level, + {{ extract_descriptor('v:certificationStandardDescriptor::string') }} as certification_standard, + {{ extract_descriptor('v:educatorRoleDescriptor::string') }} as educator_role, + {{ extract_descriptor('v:instructionalSettingDescriptor::string') }} as instructional_setting, + {{ extract_descriptor('v:minimumDegreeDescriptor::string') }} as minimum_degree, + {{ extract_descriptor('v:populationServedDescriptor::string') }} as population_served, + -- unnested lists + v:certificationExams as v_certification_exams, + v:gradeLevels as v_grade_levels, + v:routes as v_routes + from certifications +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__fieldwork_experiences.sql b/models/staging/tpdm/base/base_tpdm__fieldwork_experiences.sql new file mode 100644 index 0000000..442c3be --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__fieldwork_experiences.sql @@ -0,0 +1,30 @@ +with fieldwork_experiences as ( + {{ source_edfi3('fieldwork_experiences') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:beginDate::date as begin_date, + v:fieldworkIdentifier::string as fieldwork_id, + v:studentReference:studentUniqueId::string as student_unique_id, + -- non-identity components + v:educatorPreparationProgramReference:educationOrganizationId::int as ed_org_id, + v:educatorPreparationProgramReference:link:rel::string as ed_org_type, + v:endDate::date as end_date, + -- descriptors + {{ extract_descriptor('v:fieldworkTypeDescriptor::string') }} as fieldwork_type, + -- references + v:educatorPreparationProgramReference as educator_preparation_program_reference, + v:studentReference as student_reference + from fieldwork_experiences +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_response_education_organization_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_response_education_organization_target_associations.sql new file mode 100644 index 0000000..d9e4454 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_response_education_organization_target_associations.sql @@ -0,0 +1,24 @@ +with survey_response_education_organization_target_associations as ( + {{ source_edfi3('survey_response_education_organization_target_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + v:educationOrganizationReference:link:rel::string as ed_org_type, + v:surveyResponseReference:surveyIdentifier::string as survey_id, + v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, + -- references + v:educationOrganizationReference as education_organization_reference, + v:surveyResponseReference as survey_response_reference + from survey_education_organization_responses +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_response_person_target_associations.sql index 41be096..9de215a 100644 --- a/models/staging/tpdm/base/base_tpdm__survey_response_person_target_associations.sql +++ b/models/staging/tpdm/base/base_tpdm__survey_response_person_target_associations.sql @@ -13,11 +13,11 @@ renamed as ( v:id::string as record_guid, -- identity components - v:surveyResponseReference:namespace::string as namespace, - v:personReference:personId::string as person_id, - {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string')}} as source_system, - v:surveyResponseReference:surveyIdentifier::string as survey_id, - v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, + v:surveyResponseReference:namespace::string as namespace, + v:personReference:personId::string as person_id, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as source_system, + v:surveyResponseReference:surveyIdentifier::string as survey_id, + v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, -- references v:surveyResponseReference as survey_response_reference, v:personReference as person_reference, diff --git a/models/staging/tpdm/stage/_tpdm__stage.yml b/models/staging/tpdm/stage/_tpdm__stage.yml index 0faf0ea..0d85745 100644 --- a/models/staging/tpdm/stage/_tpdm__stage.yml +++ b/models/staging/tpdm/stage/_tpdm__stage.yml @@ -13,6 +13,23 @@ referential_integrity_tests: field: k_educator_prep_program tags: ['ref_integrity'] + - k_applicant_profile: &ref_k_applicant_profile + - relationships: + to: ref('stg_tpdm__applicant_profiles') + field: k_applicant_profile + tags: ['ref_integrity'] + + - k_staff: &ref_k_staff + - relationships: + to: ref('stg_ef3__staffs') + field: k_staff + tags: ['ref_integrity'] + + - k_certification_exam: &ref_k_certification_exam + - relationships: + to: ref('stg_tpdm__certification_exam') + field: k_certification_exam + - k_course_section: &ref_k_course_section - relationships: to: ref('stg_ef3__sections') @@ -30,7 +47,7 @@ referential_integrity_tests: to: ref('stg_ef3__students') field: k_student tags: ['ref_integrity'] - + - k_survey_response: &ref_k_survey_response - relationships: to: ref('stg_ef3__survey_responses') @@ -102,6 +119,56 @@ models: config: tags: ['tpdm'] + - name: stg_tpdm__applicant_profiles + config: + tags: ['tpdm'] + + - name: stg_tpdm__applications + config: + tags: ['tpdm'] + columns: + - name: k_applicant_profile + tests: + *ref_k_applicant_profile + + - name: stg_tpdm__candidate_relationship_to_staff_associations + config: + tags: ['tpdm'] + columns: + - name: k_candidate + tests: + *ref_k_candidate + - name: k_staff + tests: + *ref_k_staff + + - name: stg_tpdm__certification_exam_results + config: + tags: ['tpdm'] + columns: + - name: k_person + tests: + *ref_k_person + - name: k_certification_exam + tests: + *ref_k_certification_exam + + - name: stg_tpdm__certification_exams + config: + tags: ['tpdm'] + + - name: stg_tpdm__certifications + config: + tags: ['tpdm'] + + - name: stg_tpdm__educator_preparation_programs + config: + tags: ['tpdm'] + columns: + - name: k_educator_prep_program + tests: + *ref_k_educator_prep_program + - name: stg_tpdm__evaluation_element_ratings config: tags: ['tpdm'] @@ -170,6 +237,28 @@ models: tests: *ref_k_student + - name: stg_tpdm__educator_preparation_programs + config: + tags: ['tpdm'] + columns: + - name: k_educator_prep_program + tests: + *ref_k_educator_prep_program + - name: k_student + tests: + *ref_k_student + + - name: stg_tpdm__candidate_relationship_to_staff_associations + config: + tags: ['tpdm'] + columns: + - name: k_survey_response + tests: + *ref_k_survey_response + - name: k_person + tests: + *ref_k_person + - name: stg_tpdm__performance_evaluation_ratings config: tags: ['tpdm'] diff --git a/models/staging/tpdm/stage/stg_tpdm__applicant_profiles.sql b/models/staging/tpdm/stage/stg_tpdm__applicant_profiles.sql new file mode 100644 index 0000000..9b7e8de --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__applicant_profiles.sql @@ -0,0 +1,24 @@ +with base_applicant_profiles as ( + select * from {{ ref('base_tpdm__applicant_profiles') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'lower(applicant_profile_id)'] + ) }} as k_applicant_profile, + base_applicant_profiles.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_applicant_profiles +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_applicant_profile', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/stage/stg_tpdm__applications.sql b/models/staging/tpdm/stage/stg_tpdm__applications.sql new file mode 100644 index 0000000..45661ca --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__applications.sql @@ -0,0 +1,28 @@ +with applications as ( + select * from {{ ref('base_tpdm__applications') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'lower(applicant_profile_id)', + 'lower(application_id)', + 'ed_org_id'] + ) }} as k_application, + {{ edorg_ref(annualize=False) }}, + {{ gen_skey('k_applicant_profile')}}, + applications.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from applications +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_application', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/stage/stg_tpdm__candidate_relationship_to_staff_associations.sql b/models/staging/tpdm/stage/stg_tpdm__candidate_relationship_to_staff_associations.sql new file mode 100644 index 0000000..414f7a8 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__candidate_relationship_to_staff_associations.sql @@ -0,0 +1,27 @@ +with candidate_relationship_to_staff_associations as ( + select * from {{ ref('base_tpdm__candidate_relationship_to_staff_associations') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'lower(candidate_id)', + 'lower(staff_unique_id)'] + ) }} as k_candidate_relationship_to_staff_associations, + {{ gen_skey('k_candidate') }}, + {{ gen_skey('k_staff') }}, + candidate_relationship_to_staff_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from candidate_relationship_to_staff_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_candidate_relationship_to_staff_associations', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/stage/stg_tpdm__certification_exam_results.sql b/models/staging/tpdm/stage/stg_tpdm__certification_exam_results.sql new file mode 100644 index 0000000..a8bcbd3 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__certification_exam_results.sql @@ -0,0 +1,30 @@ +with certification_exam_results as ( + select * from {{ ref('base_tpdm__certification_exam_results') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'certification_exam_date', + 'lower(certification_exam_id)', + 'lower(namespace)', + 'lower(person_id)', + 'lower(source_system)'] + ) }} as k_certification_exam_result, + {{ gen_skey('k_person') }}, + {{ gen_skey('k_certification_exam') }}, + certification_exam_results.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from certification_exam_results +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_certification_exam_result', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/stage/stg_tpdm__certification_exams.sql b/models/staging/tpdm/stage/stg_tpdm__certification_exams.sql new file mode 100644 index 0000000..a384c0a --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__certification_exams.sql @@ -0,0 +1,26 @@ +with certification_exams as ( + select * from {{ ref('base_tpdm__certification_exams') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'certification_exam_date', + 'lower(certification_exam_id)', + 'lower(namespace)'] + ) }} as k_certification_exam, + certification_exams.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from certification_exams +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_certification_exam', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/stage/stg_tpdm__certifications.sql b/models/staging/tpdm/stage/stg_tpdm__certifications.sql new file mode 100644 index 0000000..5563533 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__certifications.sql @@ -0,0 +1,26 @@ +with certifications as ( + select * from {{ ref('base_tpdm__certifications') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'certification_exam_date', + 'lower(certification_exam_id)', + 'lower(namespace)'] + ) }} as k_certification, + certifications.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from certifications +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_certification', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/stage/stg_tpdm__fieldwork_experiences.sql b/models/staging/tpdm/stage/stg_tpdm__fieldwork_experiences.sql new file mode 100644 index 0000000..43b9e71 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__fieldwork_experiences.sql @@ -0,0 +1,29 @@ +with fieldwork_experiences as ( + select * from {{ ref('base_tpdm__fieldwork_experiences') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + ['tenant_code', + 'api_year', + 'begin_date', + 'lower(fieldwork_id)', + 'lower(student_unique_id)'] + ) }} as k_fieldwork_experience, + {{ gen_skey('k_educator_prep_program') }}, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + fieldwork_experiences.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from fieldwork_experiences +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_fieldwork_experience', + order_by='pull_timestamp desc') + }} +) +select * from deduped diff --git a/models/staging/tpdm/stage/stg_tpdm__survey_response_person_target_associations.sql b/models/staging/tpdm/stage/stg_tpdm__survey_response_person_target_associations.sql index f1098d9..77ebbeb 100644 --- a/models/staging/tpdm/stage/stg_tpdm__survey_response_person_target_associations.sql +++ b/models/staging/tpdm/stage/stg_tpdm__survey_response_person_target_associations.sql @@ -1,33 +1,30 @@ -with base_survey_response_person_target_associations as ( +with survey_response_person_target_associations as ( select * from {{ ref('base_tpdm__survey_response_person_target_associations') }} where not is_deleted ), keyed as ( - select + select {{ dbt_utils.surrogate_key( - [ - 'tenant_code', - 'api_year', - 'lower(namespace)', - 'lower(person_id)', - 'lower(source_system)', - 'lower(survey_id)', - 'lower(survey_response_id)' - ] + ['tenant_code', + 'api_year', + 'lower(namespace)', + 'lower(person_id)', + 'lower(source_system)', + 'lower(survey_id)', + 'lower(survey_response_id)'] ) }} as k_survey_response_person_target_association, - {{ gen_skey('k_person') }}, {{ gen_skey('k_survey_response') }}, - base_survey_response_person_target_associations.* + {{ gen_skey('k_person') }}, + survey_response_person_target_associations.* {{ extract_extension(model_name=this.name, flatten=True) }} - from base_survey_response_person_target_associations + from survey_response_person_target_associations ), deduped as ( {{ dbt_utils.deduplicate( relation='keyed', partition_by='k_survey_response_person_target_association', - order_by='pull_timestamp desc' - ) + order_by='pull_timestamp desc') }} ) select * from deduped