Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle external bigquery relations (#791) #828

Merged
merged 5 commits into from
Jul 11, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

### Changes

- Support external BigQuery relations ([#828](https://github.com/fishtown-analytics/dbt/pull/828))
- Added tox environments that have the user specify what tests should be run ([#837](https://github.com/fishtown-analytics/dbt/pull/837))
- Set the TCP keepalive on redshift ([#826](https://github.com/fishtown-analytics/dbt/pull/826))
- Fix the error handling for profiles.yml validation ([#820](https://github.com/fishtown-analytics/dbt/pull/820))
- Make the `--threads` parameter actually change the number of threads used ([#819](https://github.com/fishtown-analytics/dbt/pull/819))
- Use Mapping instead of dict as the base class for APIObject ([#756](https://github.com/fishtown-analytics/dbt/pull/756))
Expand Down
3 changes: 2 additions & 1 deletion dbt/adapters/bigquery/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@


class BigQueryRelation(DefaultRelation):
External = "external"

DEFAULTS = {
'metadata': {
Expand Down Expand Up @@ -54,7 +55,7 @@ class BigQueryRelation(DefaultRelation):
},
},
'type': {
'enum': DefaultRelation.RelationTypes + [None],
'enum': DefaultRelation.RelationTypes + [External, None],
},
'path': PATH_SCHEMA,
'include_policy': POLICY_SCHEMA,
Expand Down
86 changes: 86 additions & 0 deletions test/unit/test_bigquery_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import dbt.flags as flags

from dbt.adapters.bigquery import BigQueryAdapter
from dbt.adapters.bigquery.relation import BigQueryRelation
import dbt.exceptions
from dbt.logger import GLOBAL_LOGGER as logger # noqa

Expand Down Expand Up @@ -58,3 +59,88 @@ def test_acquire_connection_service_account_validations(self, mock_open_connecti
self.fail('validation failed with unknown exception: {}'.format(str(e)))

mock_open_connection.assert_called_once()


class TestBigQueryRelation(unittest.TestCase):
def setUp(self):
flags.STRICT_MODE = True

def test_view_temp_relation(self):
kwargs = {
'type': None,
'path': {
'project': 'test-project',
'schema': 'test_schema',
'identifier': 'my_view'
},
'table_name': 'my_view__dbt_tmp',
'quote_policy': {
'identifier': False
}
}
BigQueryRelation(**kwargs)

def test_view_relation(self):
kwargs = {
'type': 'view',
'path': {
'project': 'test-project',
'schema': 'test_schema',
'identifier': 'my_view'
},
'table_name': 'my_view',
'quote_policy': {
'identifier': True,
'schema': True
}
}
BigQueryRelation(**kwargs)

def test_table_relation(self):
kwargs = {
'type': 'table',
'path': {
'project': 'test-project',
'schema': 'test_schema',
'identifier': 'generic_table'
},
'table_name': 'generic_table',
'quote_policy': {
'identifier': True,
'schema': True
}
}
BigQueryRelation(**kwargs)

def test_external_source_relation(self):
kwargs = {
'type': 'external',
'path': {
'project': 'test-project',
'schema': 'test_schema',
'identifier': 'sheet'
},
'table_name': 'sheet',
'quote_policy': {
'identifier': True,
'schema': True
}
}
BigQueryRelation(**kwargs)

def test_invalid_relation(self):
kwargs = {
'type': 'invalid-type',
'path': {
'project': 'test-project',
'schema': 'test_schema',
'identifier': 'my_invalid_id'
},
'table_name': 'my_invalid_id',
'quote_policy': {
'identifier': False,
'schema': True
}
}
with self.assertRaises(dbt.exceptions.ValidationException):
BigQueryRelation(**kwargs)