Skip to content
This repository has been archived by the owner on Aug 16, 2022. It is now read-only.

feat: Testable Migrations #118

Merged
merged 2 commits into from
Jan 23, 2022
Merged

feat: Testable Migrations #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d1de94
feat: Testable Migrations
disq Jan 20, 2022
aa8d125
Update SDK
disq Jan 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 12 additions & 33 deletions .github/workflows/test_migration.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ jobs:
id: cache-migrations
uses: actions/cache@v2
with:
path: resources/migrations
key: ${{ runner.os }}-${{ hashFiles('resources/migrations') }}
path: resources/provider/migrations
key: ${{ runner.os }}-${{ hashFiles('resources/provider/migrations') }}

test_migration:
needs: should_run
Expand All @@ -32,7 +32,7 @@ jobs:
matrix:
dbversion: [ "postgres:latest" ]
go: [ "1.17" ]
platform: [ ubuntu-latest ] # can not run in macOS and widnowsOS
platform: [ ubuntu-latest ] # can not run in macOS and windowsOS
runs-on: ${{ matrix.platform }}
services:
postgres:
Expand All @@ -49,7 +49,6 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5

steps:
- name: Set up Go 1.x
uses: actions/setup-go@v2
Expand All @@ -59,41 +58,21 @@ jobs:
- name: Check out code into the Go module directory
uses: actions/checkout@v2

- name: Cache CQ
id: cache-cq-binary
uses: actions/cache@v2
with:
path: cloudquery
key: ${{ runner.os }}-${{ hashFiles('cloudquery') }}

- name: Download Cloudquery
if: steps.cache-cq-binary.cache-hit != 'true'
run: |
curl -L https://github.com/cloudquery/cloudquery/releases/latest/download/cloudquery_${OS}_x86_64 -o cloudquery
chmod a+x cloudquery
env:
OS: Linux

- name: Build initial schema
run: |
CQ_NO_TELEMETRY=1 ./cloudquery provider build-schema azure --config ./client/testdata/initial_migration.hcl --enable-console-log

- name: Get dependencies
run: |
go get -v -t -d ./...

- name: Build
run: go build -v .
- name: Test migrations for Postgres
run: |
go test -v ./resources/provider/provider_test.go
env:
CQ_MIGRATION_TEST_DSN: postgres://postgres:pass@localhost:5432/postgres?sslmode=disable

- name: Run upgrade
- name: Test migrations for Timescale
run: |
rm -rf .cq_reattach
go run main.go & while [ ! -f .cq_reattach ]; do sleep 1; done && \
./cloudquery provider upgrade azure --config ./client/testdata/default.hcl --enable-console-log
go test -v ./resources/provider/provider_test.go
env:
CQ_PROVIDER_DEBUG: 1
CQ_REATTACH_PROVIDERS: .cq_reattach
CQ_NO_TELEMETRY: 1
CQ_MIGRATION_TEST_DSN: tsdb://postgres:pass@localhost:5432/postgres?sslmode=disable

- name: Slack Notification
uses: rtCamp/action-slack-notify@v2
Expand All @@ -103,4 +82,4 @@ jobs:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: 'Azure - migration test failed'
SLACK_TITLE: Azure - migration test failed
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
4 changes: 2 additions & 2 deletions client/testing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func AzureMockTestHelper(t *testing.T, table *schema.Table, builder func(*testin

providertest.TestResource(t, providertest.ResourceTestCase{
Provider: &provider.Provider{
Name: "aws_mock_test_provider",
Name: "azure_mock_test_provider",
Version: "development",
Configure: func(logger hclog.Logger, i interface{}) (schema.ClientMeta, error) {
c := NewAzureClient(logging.New(&hclog.LoggerOptions{
Expand All @@ -59,7 +59,7 @@ func AzureTestHelper(t *testing.T, table *schema.Table) {

providertest.TestResource(t, providertest.ResourceTestCase{
Provider: &provider.Provider{
Name: "aws_mock_test_provider",
Name: "azure_mock_test_provider",
Version: "development",
Configure: Configure,
Config: func() provider.Config {
Expand Down
6 changes: 2 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ require (
github.com/Azure/azure-sdk-for-go v58.0.0+incompatible
github.com/Azure/go-autorest/autorest v0.11.21
github.com/Azure/go-autorest/autorest/azure/auth v0.5.7
github.com/cloudquery/cq-provider-sdk v0.6.1
github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2
github.com/cloudquery/faker/v3 v3.7.5
github.com/golang/mock v1.6.0
github.com/hashicorp/go-hclog v1.0.0
github.com/stretchr/testify v1.7.0
github.com/tombuildsstuff/giovanni v0.16.0
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
)

require (
Expand Down Expand Up @@ -44,8 +45,6 @@ require (
github.com/hashicorp/go-version v1.3.0 // indirect
github.com/hashicorp/hcl/v2 v2.10.1 // indirect
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect
github.com/huandu/go-sqlbuilder v1.13.0 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/iancoleman/strcase v0.2.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.10.0 // indirect
Expand Down Expand Up @@ -83,7 +82,6 @@ require (
go.uber.org/atomic v1.6.0 // indirect
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect
golang.org/x/text v0.3.6 // indirect
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12 // indirect
Expand Down
14 changes: 4 additions & 10 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ
github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cloudquery/cq-provider-sdk v0.6.1 h1:pyHabGR81AdsnwtZF0oaJhF9VPK5tHiC8DukA5JEW/A=
github.com/cloudquery/cq-provider-sdk v0.6.1/go.mod h1:lLjzStk8uqMiunTDnAp26QXyQ3XAMexOqzuo8T2riMc=
github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2 h1:GY0NJLEYf5JSHluVJsdAfFN00ygX5A+HZHw6/LDif5Q=
github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs=
github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
github.com/cloudquery/faker/v3 v3.7.5 h1:G7ANdEEcm8TvAAjIwNWSLrYK36CFCiSlrCqOTGCccL0=
github.com/cloudquery/faker/v3 v3.7.5/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
Expand Down Expand Up @@ -250,8 +250,8 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7
github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4=
github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
Expand Down Expand Up @@ -605,12 +605,6 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 h1:brI5vBRUlAlM34VFmnLPwjnCL/FxAJp9XvOdX6Zt+XE=
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
github.com/huandu/go-sqlbuilder v1.13.0 h1:IN1VRzcyQ+Kx74L0g5ZAY5qDaRJjwMWVmb6GrFAF8Jc=
github.com/huandu/go-sqlbuilder v1.13.0/go.mod h1:LILlbQo0MOYjlIiGgOSR3UcWQpd5Y/oZ7HLNGyAUz0E=
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
Expand Down
8 changes: 0 additions & 8 deletions resources/provider/migrations/1_v0.3.8.down.sql
View file
Open in desktop

This file was deleted.

8 changes: 0 additions & 8 deletions resources/provider/migrations/1_v0.3.8.up.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions resources/provider/migrations/2_v0.3.9.down.sql
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions resources/provider/migrations/2_v0.3.9.up.sql
View file
Open in desktop

This file was deleted.

92 changes: 0 additions & 92 deletions resources/provider/migrations/3_v0.3.11.down.sql
View file
Open in desktop

This file was deleted.

67 changes: 0 additions & 67 deletions resources/provider/migrations/3_v0.3.11.up.sql
View file
Open in desktop

This file was deleted.

Loading