Merge branch 'master' into various-data-quality-improvements

.github/workflows/data_tests.yml

@@ -0,0 +1,43 @@
+name: Data Tests
+
+on: [push, pull_request]
+
+jobs:
+  data_tests:
+    name: Data tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [duplicate_entries, file_format, missing_values, vote_breakdown_totals]
+
+    env:
+      LOG_FILE: ${{ github.workspace }}/${{ matrix.test }}.txt
+
+    steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+
+    - name: Check out data
+      uses: actions/checkout@v4
+      with:
+        path: data
+
+    - name: Check out data tests
+      uses: actions/checkout@v4
+      with:
+        repository: openelections/openelections-data-tests
+        ref: v2.2.0
+        path: data_tests
+
+    - name: Run data tests
+      run: python3 ${{ github.workspace }}/data_tests/run_tests.py --group-failures --log-file=${{ env.LOG_FILE }} ${{ matrix.test }} ${{ github.workspace }}/data
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: ${{ matrix.test }}_full_logs
+        path: ${{ env.LOG_FILE }}

.github/workflows/data_tests_changed_files.yml

@@ -0,0 +1,65 @@
+name: Data Tests (Changed Files)
+
+on:
+  pull_request:
+
+jobs:
+  data_tests:
+    name: Data tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [duplicate_entries, file_format, missing_values, vote_breakdown_totals]
+
+    env:
+      LOG_FILE: ${{ github.workspace }}/${{ matrix.test }}.txt
+
+    steps:
+    - name: Save the pull request number in an artifact
+      shell: bash
+      run: echo ${{ github.event.number }} > pull_request_number.txt
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+
+    - name: Check out data
+      uses: actions/checkout@v4
+      with:
+        path: data
+        fetch-depth: 0
+
+    - name: Check out data tests
+      uses: actions/checkout@v4
+      with:
+        repository: openelections/openelections-data-tests
+        ref: v2.2.0
+        path: data_tests
+
+    - name: Get changed *.csv files
+      id: changed-files
+      uses: tj-actions/changed-files@v44
+      with:
+        path: data
+        files: |
+          **/*.csv
+
+    - name: Run data tests
+      if: "${{ steps.changed-files.outputs.added_files != '' }}"
+      run: python3 ${{ github.workspace }}/data_tests/run_tests.py --files ${{ steps.changed-files.outputs.added_files }} --group-failures --log-file=${{ env.LOG_FILE }} --truncate-log-file ${{ matrix.test }} ${{ github.workspace }}/data
+
+    - name: Upload the pull request number
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: pull_request_number
+        path: ./pull_request_number.txt
+
+    - name: Upload error logs
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: ${{ matrix.test }}
+        path: ${{ env.LOG_FILE }}

.github/workflows/pull_request_comment.yml

@@ -0,0 +1,56 @@
+name: Comment on Pull Request
+
+on:
+  workflow_run:
+    workflows: [Data Tests (Changed Files)]
+    types:
+      - completed
+
+jobs:
+  comment:
+    name: Comment on pull request
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'failure'
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [duplicate_entries, file_format, missing_values, vote_breakdown_totals]
+
+    steps:
+      - name: Download artifacts
+        uses: dawidd6/action-download-artifact@v3
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+
+      - name: Read pull request number
+        id: pull_request_number_reader
+        uses: juliangruber/read-file-action@v1
+        with:
+          path: ./pull_request_number/pull_request_number.txt
+
+      - name: Check for failures
+        id: check_failures
+        uses: andstor/file-existence-action@v3
+        with:
+          files: ./${{ matrix.test }}/${{ matrix.test }}.txt
+
+      - name: Read failure logs
+        if: steps.check_failures.outputs.files_exists == 'true'
+        id: failure_reader
+        uses: juliangruber/read-file-action@v1
+        with:
+          path: ./${{ matrix.test }}/${{ matrix.test }}.txt
+
+      - name: Comment on pull request
+        if: steps.check_failures.outputs.files_exists == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: ${{ steps.pull_request_number_reader.outputs.content }},
+              body: `Thank you for your contribution!  Our <code>${{ matrix.test }}</code> `+
+                `test detected some potential issues in the files you added.\n\n`+
+                `<pre><code>${{ steps.failure_reader.outputs.content }}</code></pre>`
+            });

.github/workflows/tests.yml

@@ -1,10 +1,6 @@
-name: Tests
+name: Unit Tests
 
-on:
-  push:
-    branches: "*"
-  pull_request:
-    branches: "*"
+on: [push, pull_request]
 
 jobs:
   unit_tests:
@@ -15,13 +11,16 @@ jobs:
 
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+
 
     - name: Check out source
-      uses: actions/checkout@v2.3.4
+      uses: actions/checkout@v3
 
     - name: Configure environment
       run: pip install -r requirements.txt
 
     - name: Run unit tests
-      run: pytest --tb=line --color=yes
+      run: pytest --tb=short --color=yes

2014/20140603__ca__primary__fresno__precinct.csv

@@ -5029,7 +5029,7 @@ Fresno,0000082,Treasurer,,REP,Greg Conlon,265
 Fresno,0000082,Treasurer,,DEM,John Chiang,145
 Fresno,0000082,U.S. House,4,IND,Jeffrey D. Gerlach,62
 Fresno,0000082,U.S. House,4,REP,Tom McClintock,287
-Fresno,0000079,U.S. House,4,,"Arthur ""Art"" Moore",65
+Fresno,0000082,U.S. House,4,,"Arthur ""Art"" Moore",65
 Fresno,0000083,Attorney General,,REP,David King,31
 Fresno,0000083,Attorney General,,REP,John Haggerty,30
 Fresno,0000083,Attorney General,,LIB,Jonathan Jaech,6

2014/20140603__ca__primary__kern__precinct.csv

@@ -13221,7 +13221,7 @@ Kern,0040443,Treasurer,,GRN,Ellen H. Brown,6
 Kern,0040443,Treasurer,,REP,Greg Conlon,101
 Kern,0040443,Treasurer,,DEM,John Chiang,44
 Kern,0040443,U.S. House,23,REP,Kevin McCarthy,129
-Kern,0040441,U.S. House,23,DEM,Raul Garcia,1
+Kern,0040443,U.S. House,23,DEM,Raul Garcia,1
 Kern,0040444,Attorney General,,REP,David King,3
 Kern,0040444,Attorney General,,REP,John Haggerty,4
 Kern,0040444,Attorney General,,LIB,Jonathan Jaech,2