forked from great-expectations/great_expectations
-
Notifications
You must be signed in to change notification settings - Fork 0
/
azure-pipelines-gcp-integration.yml
119 lines (106 loc) · 4.6 KB
/
azure-pipelines-gcp-integration.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
stages:
- stage: cloud_db_integration_and_performance_tests
pool:
vmImage: 'ubuntu-latest'
jobs:
- job: bigquery_performance_test
timeoutInMinutes: 30 # this should be more than sufficient since the performance typically runs < 5 min
variables:
python.version: '3.8'
strategy:
matrix:
performance:
test_script: 'tests/performance/test_bigquery_benchmarks.py'
extra_args: '--bigquery --performance-tests --benchmark-json=junit/benchmark.json -k test_taxi_trips_benchmark[1-True-V3] '
maxParallel: 1
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'
- bash: python -m pip install --upgrade pip==20.2.4
displayName: 'Update pip'
- script: |
pip install --requirement requirements-dev.txt --constraint constraints-dev.txt
displayName: 'Install dependencies'
- script: |
pip install google-cloud-bigquery-storage
displayName: 'Install Google Cloud dependencies'
- task: DownloadSecureFile@1
name: gcp_authkey
displayName: 'Download Google Service Account'
inputs:
secureFile: 'superconductive-service-acct_ge-oss-ci-cd.json'
retryCount: '2'
- script: |
pip install pytest-azurepipelines
pip freeze > pip-freeze.txt
mkdir -p junit
pytest -v $(test_script) \
$(extra_args) \
--bigquery \
--junitxml=junit/test-results.xml \
--napoleon-docstrings --cov=. --cov-report=xml --cov-report=html \
--ignore=tests/cli --ignore=tests/integration/usage_statistics
displayName: 'pytest'
env:
GOOGLE_APPLICATION_CREDENTIALS: $(gcp_authkey.secureFilePath)
GE_TEST_GCP_PROJECT: $(GE_TEST_GCP_PROJECT)
GE_TEST_BIGQUERY_DATASET: $(GE_TEST_BIGQUERY_DATASET)
GE_TEST_BIGQUERY_PERFORMANCE_DATASET: $(GE_TEST_BIGQUERY_PERFORMANCE_DATASET)
- task: PublishTestResults@2
inputs:
searchFolder: junit
testResultsFiles: test-results.xml
- publish: junit/benchmark.json
artifact: BenchmarkResult
# The pip freeze output could be helpful to reproduce performance test results.
- publish: pip-freeze.txt
artifact: PipFreeze
- job: bigquery_expectations_test
timeoutInMinutes: 150 # Each stage runs in about 60 min and 30 min respectively.
variables:
python.version: '3.8'
strategy:
matrix:
expectations_v3_api:
test_script: 'tests/test_definitions/test_expectations_v3_api.py'
extra_args: ''
expectations_v2_api:
test_script: 'tests/test_definitions/test_expectations_v2_api.py'
extra_args: ''
maxParallel: 1
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'
- bash: python -m pip install --upgrade pip==20.2.4
displayName: 'Update pip'
- script: |
pip install --requirement requirements-dev.txt --constraint constraints-dev.txt
displayName: 'Install dependencies'
- script: |
pip install google-cloud-bigquery-storage
displayName: 'Install Google Cloud dependencies'
- task: DownloadSecureFile@1
name: gcp_authkey
displayName: 'Download Google Service Account'
inputs:
secureFile: 'superconductive-service-acct_ge-oss-ci-cd.json'
retryCount: '2'
- script: |
pip install pytest-azurepipelines
pip freeze > pip-freeze.txt
mkdir -p junit
pytest -v $(test_script) \
$(extra_args) \
--bigquery \
--junitxml=junit/test-results.xml \
--napoleon-docstrings --cov=. --cov-report=xml --cov-report=html \
--ignore=tests/cli --ignore=tests/integration/usage_statistics
displayName: 'pytest'
env:
GOOGLE_APPLICATION_CREDENTIALS: $(gcp_authkey.secureFilePath)
GE_TEST_GCP_PROJECT: $(GE_TEST_GCP_PROJECT)
GE_TEST_BIGQUERY_DATASET: $(GE_TEST_BIGQUERY_DATASET)