This repository has been archived by the owner on May 6, 2024. It is now read-only.
Run district scrapers #30321
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run district scrapers | |
| on: | |
| schedule: | |
| - cron: '10 * * * *' # run every hour at xx:10 | |
| workflow_dispatch: ~ | |
| jobs: | |
| run_scraper: | |
| runs-on: ubuntu-20.04 | |
| continue-on-error: false | |
| timeout-minutes: 10 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| canton: | |
| #- AG | |
| - BE | |
| #- BL | |
| #- FR | |
| #- GR | |
| - SG | |
| #- SO | |
| #- SZ | |
| - TG | |
| #- VS | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.7 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.7 | |
| - run: npm ci | |
| - name: Remove broken apt repos | |
| run: | | |
| for apt_file in `grep -lr microsoft /etc/apt/sources.list.d/`; do sudo rm $apt_file; done | |
| - name: Install dependencies | |
| env: | |
| SCRAPER_KEY: ${{ matrix.canton }} | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| pip install -r requirements.txt | |
| sudo apt update || true # do not fail if update does not work | |
| sudo apt-get install sqlite3 poppler-utils | |
| if [ "$SCRAPER_KEY" = "AG" ] ; then | |
| pip install -r requirements-ocr.txt | |
| sudo apt-get install tesseract-ocr=3.04.01-4 | |
| fi | |
| - name: Scrape new data | |
| env: | |
| SCRAPER_KEY: ${{ matrix.canton }} | |
| run: | | |
| ./scrapers/run_district_scraper.sh | |
| - name: Check if there are changes in the repo | |
| run: | | |
| if git diff -w --no-ext-diff --quiet | |
| then | |
| echo "changed=0" >> $GITHUB_OUTPUT | |
| else | |
| echo "changed=1" >> $GITHUB_OUTPUT | |
| fi | |
| id: changes | |
| - name: Set commit message | |
| env: | |
| SCRAPER_KEY: ${{ matrix.canton }} | |
| run: | | |
| echo "commit_msg=Update fallzahlen_kanton_${SCRAPER_KEY}_bezirk.csv from scraper" >> $GITHUB_ENV | |
| - name: Commit and push to repo | |
| if: steps.changes.outputs.changed == 1 # only try to commit if there are actually changes | |
| uses: github-actions-x/[email protected] | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| push-branch: master | |
| name: GitHub Action Scraper | |
| email: [email protected] | |
| commit-message: ${{ env.commit_msg }} | |
| rebase: 'true' | |
| - name: Get current unix timestamp | |
| if: always() | |
| id: date | |
| run: echo "ts=$(date +'%s')" >> $GITHUB_OUTPUT | |
| - name: Notify slack failure | |
| if: ${{ failure() || cancelled() }} | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| uses: pullreminders/slack-action@master | |
| with: | |
| args: '{\"channel\":\"C013C0UUQ4S\", \"attachments\": [{\"fallback\": \"Job failed.\", \"color\": \"danger\", \"title\": \"Run district scrapers ${{ matrix.canton }}\", \"title_link\": \"https://github.com/openZH/covid_19/actions/runs/${{ github.run_id }}?check_suite_focus=true\", \"text\": \":x: District scraper failed\", \"footer\": \"<https://github.com/openZH/covid_19|openZH/covid_19>\", \"footer_icon\": \"https://github.com/abinoda/slack-action/raw/master/docs/app-icons/github-icon.png\", \"ts\": \"${{steps.date.outputs.ts}}\"}]}' | |