Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use adapter to define sort/dist for model #274

Merged
merged 5 commits into from
Feb 7, 2017
Merged

use adapter to define sort/dist for model #274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
63e9918
use adapter to define sort/dist for model
drewbanin Feb 2, 2017
25ec5d0
Merge branch 'development' into fix/use-adapter-sort-dist
drewbanin Feb 3, 2017
7984b7c
add test for pg sort/dist + pep8
drewbanin Feb 3, 2017
1cff39f
fix typos in sort/dist key generator
drewbanin Feb 3, 2017
f9a6b0f
Merge branch 'fix/use-adapter-sort-dist' of github.com:analyst-collec…
drewbanin Feb 3, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions dbt/adapters/redshift.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def acquire_connection(cls, profile):

@classmethod
def dist_qualifier(cls, dist):
dist_key = dist_key.strip().lower()
dist_key = dist.strip().lower()

if dist_key in ['all', 'even']:
return 'diststyle({})'.format(dist_key)
Expand All @@ -53,8 +53,10 @@ def sort_qualifier(cls, sort_type, sort):
.format(sort_type, valid_sort_types)
)

if type(sort_keys) == str:
sort_keys = [sort_keys]
if type(sort) == str:
sort_keys = [sort]
else:
sort_keys = sort

formatted_sort_keys = ['"{}"'.format(sort_key)
for sort_key in sort_keys]
Expand Down
30 changes: 5 additions & 25 deletions dbt/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import dbt.schema_tester
import dbt.project
import dbt.archival
from dbt.adapters.factory import get_adapter
from dbt.utils import deep_merge, DBTConfigKeys, compiler_error, \
compiler_warning

Expand Down Expand Up @@ -351,27 +352,8 @@ def sort_qualifier(self, model_config):

sort_type = sort_type.strip().lower()

valid_sort_types = ['compound', 'interleaved']
if sort_type not in valid_sort_types:
compiler_error(
self,
"Invalid sort_type given: {} -- must be one of {}".format(
sort_type, valid_sort_types
)
)

if type(sort_keys) == str:
sort_keys = [sort_keys]

# remove existing quotes in field name, then wrap in quotes
formatted_sort_keys = [
'"{}"'.format(sort_key.replace('"', '')) for sort_key in sort_keys
]
keys_csv = ', '.join(formatted_sort_keys)

return "{sort_type} sortkey ({keys_csv})".format(
sort_type=sort_type, keys_csv=keys_csv
)
adapter = get_adapter(self.project.run_environment())
return adapter.sort_qualifier(sort_type, sort_keys)

def dist_qualifier(self, model_config):
if 'dist' not in model_config:
Expand All @@ -390,10 +372,8 @@ def dist_qualifier(self, model_config):

dist_key = dist_key.strip().lower()

if dist_key in ['all', 'even']:
return 'diststyle {}'.format(dist_key)
else:
return 'diststyle key distkey ("{}")'.format(dist_key)
adapter = get_adapter(self.project.run_environment())
return adapter.dist_qualifier(dist_key)

def build_path(self):
build_dir = self.create_template.label
Expand Down
9 changes: 9 additions & 0 deletions test/integration/018_adapter_ddl_tests/models/materialized.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{{
config(
materialized = "table",
sort = 'first_name',
dist = 'first_name'
)
}}

select * from "adaper_ddl_018"."seed"
110 changes: 110 additions & 0 deletions test/integration/018_adapter_ddl_tests/seed.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
create table adaper_ddl_018.seed (
id BIGSERIAL PRIMARY KEY,
first_name VARCHAR(50),
last_name VARCHAR(50),
email VARCHAR(50),
gender VARCHAR(50),
ip_address VARCHAR(20)
);


insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', '[email protected]', 'Male', '59.80.20.168');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Walker', '[email protected]', 'Female', '194.121.179.35');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Ryan', '[email protected]', 'Male', '11.3.212.243');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Bonnie', 'Spencer', '[email protected]', 'Female', '216.32.196.175');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Taylor', '[email protected]', 'Male', '253.10.246.136');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jacqueline', 'Griffin', '[email protected]', 'Female', '16.13.192.220');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Arnold', '[email protected]', 'Female', '232.116.150.64');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Craig', 'Ortiz', '[email protected]', 'Male', '199.126.106.13');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Gary', 'Day', '[email protected]', 'Male', '35.81.68.186');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Wright', '[email protected]', 'Female', '236.82.178.100');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Raymond', 'Kelley', '[email protected]', 'Male', '213.65.166.67');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robinson', '[email protected]', 'Male', '72.232.194.193');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Mildred', 'Martinez', '[email protected]', 'Female', '198.29.112.5');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Arnold', '[email protected]', 'Male', '86.96.3.250');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gray', '[email protected]', 'Female', '79.218.162.245');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Garza', '[email protected]', 'Female', '21.59.100.54');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robertson', '[email protected]', 'Male', '131.134.82.96');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Hernandez', '[email protected]', 'Male', '254.196.137.72');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Julia', 'Gonzalez', '[email protected]', 'Female', '84.240.227.174');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Davis', '[email protected]', 'Male', '9.255.67.25');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Kimberly', 'Harper', '[email protected]', 'Female', '198.208.120.253');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Mark', 'Martin', '[email protected]', 'Male', '233.138.182.153');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Cynthia', 'Ruiz', '[email protected]', 'Female', '18.178.187.201');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Samuel', 'Carroll', '[email protected]', 'Male', '128.113.96.122');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Larson', '[email protected]', 'Female', '98.234.85.95');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Ashley', 'Perry', '[email protected]', 'Female', '247.173.114.52');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Howard', 'Rodriguez', '[email protected]', 'Male', '231.188.95.26');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Amy', 'Brooks', '[email protected]', 'Female', '141.199.174.118');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Louise', 'Warren', '[email protected]', 'Female', '96.105.158.28');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Tina', 'Watson', '[email protected]', 'Female', '251.142.118.177');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Janice', 'Kelley', '[email protected]', 'Female', '239.167.34.233');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Terry', 'Mccoy', '[email protected]', 'Male', '117.201.183.203');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jeffrey', 'Morgan', '[email protected]', 'Male', '78.101.78.149');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Louis', 'Harvey', '[email protected]', 'Male', '51.50.0.167');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Miller', '[email protected]', 'Male', '103.255.222.110');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Marshall', '[email protected]', 'Male', '149.219.91.68');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Patrick', 'Lopez', '[email protected]', 'Male', '250.136.229.89');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Jenkins', '[email protected]', 'Male', '7.36.112.81');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'Cruz', '[email protected]', 'Male', '32.38.98.15');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Hawkins', '[email protected]', 'Female', '135.171.129.255');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Barnes', '[email protected]', 'Male', '240.197.85.140');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Griffin', '[email protected]', 'Female', '19.29.135.24');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Sean', 'Mason', '[email protected]', 'Male', '159.219.155.249');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Payne', '[email protected]', 'Male', '235.168.199.218');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Steve', 'Cruz', '[email protected]', 'Male', '238.201.81.198');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Garcia', '[email protected]', 'Male', '25.85.10.18');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Doris', 'Lopez', '[email protected]', 'Female', '245.218.51.238');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Susan', 'Nichols', '[email protected]', 'Female', '199.99.9.61');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Ferguson', '[email protected]', 'Female', '236.241.135.21');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Pierce', '[email protected]', 'Female', '132.40.10.209');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Lawrence', 'Phillips', '[email protected]', 'Male', '72.226.82.87');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gilbert', '[email protected]', 'Female', '196.250.15.142');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Eric', 'Williams', '[email protected]', 'Male', '222.202.73.126');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Romero', '[email protected]', 'Male', '123.184.125.212');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jean', 'Wilson', '[email protected]', 'Female', '176.106.32.194');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Lori', 'Reynolds', '[email protected]', 'Female', '114.181.203.22');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Moreno', '[email protected]', 'Male', '233.249.97.60');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Steven', 'Berry', '[email protected]', 'Male', '186.193.50.50');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Shaw', '[email protected]', 'Female', '120.37.71.222');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('John', 'Stephens', '[email protected]', 'Male', '191.87.127.115');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Jacobs', '[email protected]', 'Male', '66.210.83.155');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Lawson', '[email protected]', 'Male', '54.98.36.94');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Peter', 'Morgan', '[email protected]', 'Male', '14.77.29.106');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Nicole', 'Garrett', '[email protected]', 'Female', '21.127.74.68');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Kim', '[email protected]', 'Male', '57.255.207.41');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Roberts', '[email protected]', 'Male', '222.143.131.109');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Montgomery', '[email protected]', 'Male', '76.75.111.77');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Alvarez', '[email protected]', 'Male', '58.157.186.194');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Olson', '[email protected]', 'Male', '69.65.74.135');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Morgan', '[email protected]', 'Male', '96.20.140.87');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Stanley', '[email protected]', 'Male', '163.119.217.44');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Long', '[email protected]', 'Female', '204.150.194.182');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Robert', 'Berry', '[email protected]', 'Male', '104.19.48.241');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Antonio', 'Brooks', '[email protected]', 'Male', '210.31.7.24');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Garcia', '[email protected]', 'Female', '233.218.162.214');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hanson', '[email protected]', 'Male', '31.55.46.199');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Nelson', '[email protected]', 'Female', '14.189.146.41');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jason', 'Reed', '[email protected]', 'Male', '141.189.89.255');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Coleman', '[email protected]', 'Male', '81.189.221.144');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'King', '[email protected]', 'Female', '212.123.168.231');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Johnny', 'Holmes', '[email protected]', 'Male', '177.3.93.188');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Gilbert', '[email protected]', 'Female', '199.215.169.61');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Thomas', '[email protected]', 'Male', '0.8.205.30');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Julie', 'Perry', '[email protected]', 'Female', '60.116.114.192');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Perry', '[email protected]', 'Male', '181.125.70.232');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Kenneth', 'Ruiz', '[email protected]', 'Male', '189.105.137.109');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jose', 'Morgan', '[email protected]', 'Male', '101.134.215.156');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Campbell', '[email protected]', 'Male', '102.120.215.84');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Debra', 'Collins', '[email protected]', 'Female', '90.13.153.235');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Jesse', 'Johnson', '[email protected]', 'Male', '225.178.125.53');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Elizabeth', 'Stone', '[email protected]', 'Female', '123.184.126.221');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Rogers', '[email protected]', 'Female', '98.104.132.187');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Emily', 'Dixon', '[email protected]', 'Female', '39.190.75.57');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Albert', 'Scott', '[email protected]', 'Male', '40.209.13.189');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Barbara', 'Peterson', '[email protected]', 'Female', '75.249.136.180');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Greene', '[email protected]', 'Male', '184.173.109.144');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Earl', 'Sanders', '[email protected]', 'Male', '247.34.90.117');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Brooks', '[email protected]', 'Female', '10.63.249.126');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Foster', '[email protected]', 'Male', '139.214.40.244');
insert into adaper_ddl_018.seed (first_name, last_name, email, gender, ip_address) values ('Carl', 'Meyer', '[email protected]', 'Male', '204.117.7.88');
23 changes: 23 additions & 0 deletions test/integration/018_adapter_ddl_tests/test_adapter_ddl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
from nose.plugins.attrib import attr
from test.integration.base import DBTIntegrationTest

class TestAdapterDDL(DBTIntegrationTest):

def setUp(self):
DBTIntegrationTest.setUp(self)

self.run_sql_file("test/integration/018_adapter_ddl_tests/seed.sql")

@property
def schema(self):
return "adaper_ddl_018"

@property
def models(self):
return "test/integration/018_adapter_ddl_tests/models"

@attr(type='postgres')
def test_sort_and_dist_keys_are_nops_on_postgres(self):
self.run_dbt(['run'])

self.assertTablesEqual("seed","materialized")