airflow-teradata-main-systemtest #221
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: airflow-teradata-main-systemtest | |
| on: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 6 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| pages: write | |
| id-token: write | |
| jobs: | |
| teradata-systemtest: | |
| name: "Manage Teradata Provider System Tests Dashboard" | |
| runs-on: "ubuntu-22.04" | |
| steps: | |
| - name: Show current user | |
| run: | | |
| whoami | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout Teradata Airflow Source Code" | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| persist-credentials: false | |
| - name: "Github Event and Branch Details" | |
| run: echo "The current checked out branch is main. Workflow trigged Event- ${{ github.event_name }}. The fully qualified reference (ref) branch that triggered the workflow run- ${{ github.ref }}" | |
| - name: "Install Python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.x | |
| cache: 'pip' | |
| - name: "Checkout ClearScape Scripts and Report Generation supported files" | |
| run: | | |
| git fetch | |
| git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/scripts/createTestEnvironments.sh | |
| git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/scripts/deleteTestEnvironments.sh | |
| git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/system-tests-files/report_generate_index_html.py | |
| git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/system-tests-files/report_index_template.html | |
| git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/system-tests-files/report_parse_xml.py | |
| cp .github/workflows/system-tests-files/report_generate_index_html.py . | |
| cp .github/workflows/system-tests-files/report_index_template.html . | |
| cp .github/workflows/system-tests-files/report_parse_xml.py . | |
| - name: "Download System Test Log Report" | |
| uses: dawidd6/action-download-artifact@v3 | |
| with: | |
| workflow: ci-teradata-main-systemtest-dashboard.yml | |
| workflow_conclusion: completed | |
| name: airflow-system-test-report | |
| path: . | |
| - name: "Display content of report csv" | |
| run: | | |
| cat reporttest.csv | |
| - name: "Define CSAE env name" | |
| id: define-environment-name | |
| run: | | |
| echo "env-name=airflow-ci-$(date +%s%N)" >> $GITHUB_OUTPUT | |
| - name: "Create CSAE environment" | |
| id: create-csae-environment | |
| shell: bash | |
| run: | | |
| echo Clearscape Environment name : ${{ steps.define-environment-name.outputs.env-name}} | |
| chmod 777 .github/workflows/scripts/createTestEnvironments.sh | |
| .github/workflows/scripts/createTestEnvironments.sh | |
| env: | |
| CSAE_TOKEN: ${{ secrets.CSAE_AIRFLOW_TOKEN }} | |
| CSAE_ENV_PASSWORD: ${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }} | |
| CSAE_ENV_NAME: ${{ steps.define-environment-name.outputs.env-name}} | |
| - name: "Install Airflow Teradata Provider and Run System tests" | |
| run: | | |
| echo Clearscape Environment name : ${{ steps.create-csae-environment.outputs.teradata-server-name }} | |
| sudo apt update | |
| sudo apt-get install -y apt-transport-https apt-utils build-essential ca-certificates dirmngr \ | |
| freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \ | |
| libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ | |
| libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \ | |
| software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev python3-venv python3-pip pipx | |
| pip install pytest time-machine pytest-asyncio pytest-md-report cryptography pytest-html tailer pandas jinja2 sqlalchemy hatch | |
| pip install -e ".[devel,teradata, microsoft.azure, amazon]" | |
| export SYSTEM_TESTS_ENV_ID=teradatasystemtest | |
| export AIRFLOW_CONN_TERADATA_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}' | |
| export AIRFLOW_CONN_TERADATA_LAKE='teradata://${{ vars.LAKE_ENV_USERNAME }}:${{ secrets.LAKE_ENV_PASSWORD }}@${{ vars.LAKE_SERVER_NAME }}:/${{ vars.LAKE_ENV_USERNAME }}' | |
| export AIRFLOW_CONN_TERADATA_SSL_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?sslmode=allow' | |
| export AIRFLOW_CONN_TERADATA_SP_CALL='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?tmode=TERA' | |
| export AIRFLOW_CONN_WASB_DEFAULT='wasb://${{ vars.AZURE_BLOB_ACCOUNTNAME }}:${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY_URI }}@' | |
| export AIRFLOW_CONN_AWS_DEFAULT=aws://${{ vars.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}@/?aws_session_token=${{ secrets.AWS_SESSION_TOKEN }} | |
| export AIRFLOW_VAR_AZURE_BLOB_ACCOUNTNAME=${{ vars.AZURE_BLOB_ACCOUNTNAME }} | |
| export AIRFLOW_VAR_AZURE_BLOB_ACCOUNT_SECRET_KEY=${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY }} | |
| export AIRFLOW_VAR_AWS_ACCESS_KEY_ID=${{ vars.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_ID }} | |
| export AIRFLOW_VAR_AWS_SECRET_ACCESS_KEY=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_KEY }} | |
| pytest --system=teradata --junitxml=report_test.xml tests/system/providers/teradata | |
| cat report_test.xml | |
| - name: "Delete CSAE environment" | |
| if: always() | |
| run: | | |
| chmod 777 .github/workflows/scripts/deleteTestEnvironments.sh | |
| .github/workflows/scripts/deleteTestEnvironments.sh | |
| env: | |
| CSAE_TOKEN: ${{ secrets.CSAE_AIRFLOW_TOKEN }} | |
| CSAE_ENV_NAME: ${{ steps.define-environment-name.outputs.env-name}} | |
| - name: "Generate Teradata Airflow Provider System Tests Report" | |
| if: always() | |
| run: | | |
| cat report_test.xml | |
| python report_parse_xml.py | |
| python report_generate_index_html.py | |
| mkdir teradata_report | |
| cp index.html teradata_report/index.html | |
| cat reporttest.csv | |
| - name: Upload airflow teradata system test report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: airflow-system-test-report | |
| path: | | |
| reporttest.csv | |
| report_test.xml | |
| - name: "Build GITHUB Page for Teradata Airflow Provider System Tests Report" | |
| if: always() | |
| uses: actions/jekyll-build-pages@v1 | |
| with: | |
| source: ./teradata_report | |
| destination: ./_site | |
| - name: "Upload Teradata Airflow Provider System Tests Report to GITHUB page" | |
| if: always() | |
| uses: actions/upload-pages-artifact@v2 | |
| - name: "Publish Teradata Airflow Provider System Tests Report to GITHUB page" | |
| if: always() | |
| id: deployment | |
| uses: actions/deploy-pages@v2 | |