Skip to content
This repository has been archived by the owner on Aug 16, 2022. It is now read-only.

Commit

Permalink
feat: Testable Migrations (#42)
Browse files Browse the repository at this point in the history
* feat: All migrations as files

* Update SDK

Co-authored-by: Kemal Hadimli <[email protected]>
  • Loading branch information
disq and disq authored Jan 23, 2022
1 parent ccdbc1b commit 0fa61a0
Show file tree
Hide file tree
Showing 11 changed files with 3,164 additions and 14 deletions.
85 changes: 85 additions & 0 deletions .github/workflows/test_migration.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
name: test_migration

on:
push:
branches:
- main
pull_request:
branches: [main]
workflow_dispatch:

jobs:

should_run:
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.cache-migrations.cache-hit != 'true' }}
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2

- name: Cache Migrations
id: cache-migrations
uses: actions/cache@v2
with:
path: resources/provider/migrations
key: ${{ runner.os }}-${{ hashFiles('resources/provider/migrations') }}

test_migration:
needs: should_run
if: github.repository == 'cloudquery/cq-provider-k8s' && needs.should_run.outputs.should_run == 'true'
strategy:
matrix:
dbversion: [ "postgres:latest" ]
go: [ "1.17" ]
platform: [ ubuntu-latest ] # can not run in macOS and windowsOS
runs-on: ${{ matrix.platform }}
services:
postgres:
image: ${{ matrix.dbversion }}
env:
POSTGRES_PASSWORD: pass
POSTGRES_USER: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v2
with:
go-version: ^1.17

- name: Check out code into the Go module directory
uses: actions/checkout@v2

- name: Get dependencies
run: |
go get -v -t -d ./...
- name: Test migrations for Postgres
run: |
go test -v ./resources/provider/provider_test.go
env:
CQ_MIGRATION_TEST_DSN: postgres://postgres:pass@localhost:5432/postgres?sslmode=disable

- name: Test migrations for Timescale
run: |
go test -v ./resources/provider/provider_test.go
env:
CQ_MIGRATION_TEST_DSN: tsdb://postgres:pass@localhost:5432/postgres?sslmode=disable

- name: Slack Notification
uses: rtCamp/action-slack-notify@v2
if: ${{ failure() }}
env:
SLACK_CHANNEL: oss-tests
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: 'K8S - migration test failed'
SLACK_TITLE: K8S - migration test failed
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
2 changes: 1 addition & 1 deletion client/testing.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func K8sMockTestHelper(t *testing.T, table *schema.Table, builder func(*testing.

providertest.TestResource(t, providertest.ResourceTestCase{
Provider: &provider.Provider{
Name: "aws_mock_test_provider",
Name: "k8s_mock_test_provider",
Version: "development",
Configure: func(logger hclog.Logger, _ interface{}) (schema.ClientMeta, error) {
c := &Client{
Expand Down
4 changes: 1 addition & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/cloudquery/cq-provider-k8s
go 1.17

require (
github.com/cloudquery/cq-provider-sdk v0.6.1
github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2
github.com/cloudquery/faker/v3 v3.7.5
github.com/golang/mock v1.6.0
github.com/hashicorp/go-hclog v1.0.0
Expand Down Expand Up @@ -45,8 +45,6 @@ require (
github.com/hashicorp/go-version v1.3.0 // indirect
github.com/hashicorp/hcl/v2 v2.10.1 // indirect
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect
github.com/huandu/go-sqlbuilder v1.13.0 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/iancoleman/strcase v0.2.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
Expand Down
14 changes: 4 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ
github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cloudquery/cq-provider-sdk v0.6.1 h1:pyHabGR81AdsnwtZF0oaJhF9VPK5tHiC8DukA5JEW/A=
github.com/cloudquery/cq-provider-sdk v0.6.1/go.mod h1:lLjzStk8uqMiunTDnAp26QXyQ3XAMexOqzuo8T2riMc=
github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2 h1:GY0NJLEYf5JSHluVJsdAfFN00ygX5A+HZHw6/LDif5Q=
github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs=
github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
github.com/cloudquery/faker/v3 v3.7.5 h1:G7ANdEEcm8TvAAjIwNWSLrYK36CFCiSlrCqOTGCccL0=
github.com/cloudquery/faker/v3 v3.7.5/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
Expand Down Expand Up @@ -233,8 +233,8 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7
github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4=
github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
Expand Down Expand Up @@ -592,12 +592,6 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 h1:brI5vBRUlAlM34VFmnLPwjnCL/FxAJp9XvOdX6Zt+XE=
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
github.com/huandu/go-sqlbuilder v1.13.0 h1:IN1VRzcyQ+Kx74L0g5ZAY5qDaRJjwMWVmb6GrFAF8Jc=
github.com/huandu/go-sqlbuilder v1.13.0/go.mod h1:LILlbQo0MOYjlIiGgOSR3UcWQpd5Y/oZ7HLNGyAUz0E=
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
Expand Down
104 changes: 104 additions & 0 deletions resources/provider/migrations/postgres/1_v0.4.0.down.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
-- Autogenerated by migration tool on 2022-01-19 11:52:48

-- Resource: apps.daemon_sets
DROP TABLE IF EXISTS k8s_apps_daemon_set_selector_match_expressions;
DROP TABLE IF EXISTS k8s_apps_daemon_set_status_conditions;
DROP TABLE IF EXISTS k8s_apps_daemon_sets;

-- Resource: apps.deployments
DROP TABLE IF EXISTS k8s_apps_deployment_selector_match_expressions;
DROP TABLE IF EXISTS k8s_apps_deployment_status_conditions;
DROP TABLE IF EXISTS k8s_apps_deployments;

-- Resource: apps.replica_sets
DROP TABLE IF EXISTS k8s_apps_replica_set_selector_match_expressions;
DROP TABLE IF EXISTS k8s_apps_replica_set_status_conditions;
DROP TABLE IF EXISTS k8s_apps_replica_sets;

-- Resource: apps.stateful_sets
DROP TABLE IF EXISTS k8s_apps_stateful_set_selector_match_expressions;
DROP TABLE IF EXISTS k8s_apps_stateful_set_status_conditions;
DROP TABLE IF EXISTS k8s_apps_stateful_sets;

-- Resource: batch.cron_jobs
DROP TABLE IF EXISTS k8s_batch_cron_jobs;

-- Resource: batch.jobs
DROP TABLE IF EXISTS k8s_batch_job_selector_match_expressions;
DROP TABLE IF EXISTS k8s_batch_job_status_conditions;
DROP TABLE IF EXISTS k8s_batch_jobs;

-- Resource: core.endpoints
DROP TABLE IF EXISTS k8s_core_endpoint_subset_addresses;
DROP TABLE IF EXISTS k8s_core_endpoint_subset_not_ready_addresses;
DROP TABLE IF EXISTS k8s_core_endpoint_subset_ports;
DROP TABLE IF EXISTS k8s_core_endpoint_subsets;
DROP TABLE IF EXISTS k8s_core_endpoints;

-- Resource: core.limit_ranges
DROP TABLE IF EXISTS k8s_core_limit_range_limits;
DROP TABLE IF EXISTS k8s_core_limit_ranges;

-- Resource: core.namespaces
DROP TABLE IF EXISTS k8s_meta_owner_references;
DROP TABLE IF EXISTS k8s_core_namespaces;

-- Resource: core.nodes
DROP TABLE IF EXISTS k8s_core_node_images;
DROP TABLE IF EXISTS k8s_core_node_volumes_attached;
DROP TABLE IF EXISTS k8s_core_nodes;

-- Resource: core.pods
DROP TABLE IF EXISTS k8s_core_pod_init_container_ports;
DROP TABLE IF EXISTS k8s_core_pod_init_container_envs;
DROP TABLE IF EXISTS k8s_core_pod_init_container_volume_mounts;
DROP TABLE IF EXISTS k8s_core_pod_init_container_volume_devices;
DROP TABLE IF EXISTS k8s_core_pod_init_containers;
DROP TABLE IF EXISTS k8s_core_pod_container_ports;
DROP TABLE IF EXISTS k8s_core_pod_container_envs;
DROP TABLE IF EXISTS k8s_core_pod_container_volume_mounts;
DROP TABLE IF EXISTS k8s_core_pod_container_volume_devices;
DROP TABLE IF EXISTS k8s_core_pod_containers;
DROP TABLE IF EXISTS k8s_core_pod_ephemeral_container_ports;
DROP TABLE IF EXISTS k8s_core_pod_ephemeral_container_envs;
DROP TABLE IF EXISTS k8s_core_pod_ephemeral_container_volume_mounts;
DROP TABLE IF EXISTS k8s_core_pod_ephemeral_container_volume_devices;
DROP TABLE IF EXISTS k8s_core_pod_ephemeral_containers;
DROP TABLE IF EXISTS k8s_core_pod_volumes;
DROP TABLE IF EXISTS k8s_core_pod_init_container_statuses;
DROP TABLE IF EXISTS k8s_core_pod_container_statuses;
DROP TABLE IF EXISTS k8s_core_pod_ephemeral_container_statuses;
DROP TABLE IF EXISTS k8s_core_pods;

-- Resource: core.resource_quotas
DROP TABLE IF EXISTS k8s_core_resource_quota_scope_selector_match_expressions;
DROP TABLE IF EXISTS k8s_core_resource_quotas;

-- Resource: core.service_accounts
DROP TABLE IF EXISTS k8s_core_service_account_secrets;
DROP TABLE IF EXISTS k8s_core_service_accounts;

-- Resource: core.services
DROP TABLE IF EXISTS k8s_core_service_ports;
DROP TABLE IF EXISTS k8s_core_service_load_balancer_ingress_ports;
DROP TABLE IF EXISTS k8s_core_service_load_balancer_ingresses;
DROP TABLE IF EXISTS k8s_core_service_conditions;
DROP TABLE IF EXISTS k8s_core_services;

-- Resource: networking.network_policies
DROP TABLE IF EXISTS k8s_networking_network_policy_pod_selector_match_expressions;
DROP TABLE IF EXISTS k8s_networking_network_policy_ingress_ports;
DROP TABLE IF EXISTS k8s_networking_network_policy_ingress_from;
DROP TABLE IF EXISTS k8s_networking_network_policy_ingress;
DROP TABLE IF EXISTS k8s_networking_network_policy_egress_ports;
DROP TABLE IF EXISTS k8s_networking_network_policy_egress_to;
DROP TABLE IF EXISTS k8s_networking_network_policy_egress;
DROP TABLE IF EXISTS k8s_networking_network_policies;

-- Resource: rbac.role_bindings
DROP TABLE IF EXISTS k8s_rbac_role_binding_subjects;
DROP TABLE IF EXISTS k8s_rbac_role_bindings;

-- Resource: rbac.roles
DROP TABLE IF EXISTS k8s_rbac_role_rules;
DROP TABLE IF EXISTS k8s_rbac_roles;
Loading

0 comments on commit 0fa61a0

Please sign in to comment.