Wrap redshift DROP ... CASCADE in a lock

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Ignore commented-out schema tests ([#330](https://github.com/fishtown-analytics/dbt/pull/330), [#328](https://github.com/fishtown-analytics/dbt/issues/328))
 - Fix run levels ([#343](https://github.com/fishtown-analytics/dbt/pull/343), [#340](https://github.com/fishtown-analytics/dbt/issues/340), [#338](https://github.com/fishtown-analytics/dbt/issues/338))
 - Fix concurrency, open a unique transaction per model ([#345](https://github.com/fishtown-analytics/dbt/pull/345), [#336](https://github.com/fishtown-analytics/dbt/issues/336))
+- Handle concurrent `DROP ... CASCADE`s in Redshift ([#349](https://github.com/fishtown-analytics/dbt/pull/349))
 
 ### Changes
 

dbt/adapters/redshift.py

@@ -1,7 +1,12 @@
+import multiprocessing
+
 from dbt.adapters.postgres import PostgresAdapter
 from dbt.logger import GLOBAL_LOGGER as logger  # noqa
 
 
+drop_lock = multiprocessing.Lock()
+
+
 class RedshiftAdapter(PostgresAdapter):
 
     @classmethod
@@ -42,3 +47,28 @@ def sort_qualifier(cls, sort_type, sort):
         return "{sort_type} sortkey({keys_csv})".format(
             sort_type=sort_type, keys_csv=keys_csv
         )
+
+    @classmethod
+    def drop(cls, profile, relation, relation_type, model_name=None):
+        global drop_lock
+
+        to_return = None
+
+        try:
+            drop_lock.acquire()
+
+            connection = cls.get_connection(profile, model_name)
+
+            cls.commit(connection)
+            cls.begin(profile, connection.get('name'))
+
+            to_return = super(PostgresAdapter, cls).drop(
+                profile, relation, relation_type, model_name)
+
+            cls.commit(connection)
+            cls.begin(profile, connection.get('name'))
+
+            return to_return
+
+        finally:
+            drop_lock.release()

test/integration/021_concurrency_test/models/table_a.sql

@@ -0,0 +1,7 @@
+{{
+  config(
+    materialized = "table"
+  )
+}}
+
+select * from "concurrency_021"."seed"

test/integration/021_concurrency_test/models/table_b.sql

@@ -0,0 +1,7 @@
+{{
+  config(
+    materialized = "table"
+  )
+}}
+
+select * from "concurrency_021"."seed"

test/integration/021_concurrency_test/models/view_with_conflicting_cascade.sql

@@ -0,0 +1,5 @@
+select * from {{ref('table_a')}}
+
+union all
+
+select * from {{ref('table_b')}}

test/integration/021_concurrency_test/test_concurrency.py

@@ -25,6 +25,19 @@ def test__postgres__concurrency(self):
 
         self.assertTablesEqual("seed", "view")
         self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")
+        self.assertTableDoesNotExist("invalid")
+        self.assertTableDoesNotExist("skip")
+
+        self.run_sql_file("test/integration/021_concurrency_test/update.sql")
+
+        self.run_dbt()
+
+        self.assertTablesEqual("seed", "view")
+        self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")
         self.assertTableDoesNotExist("invalid")
         self.assertTableDoesNotExist("skip")
 
@@ -38,3 +51,14 @@ def test__snowflake__concurrency(self):
 
         self.assertTablesEqual("seed", "view")
         self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")
+
+        self.run_sql_file("test/integration/021_concurrency_test/update.sql")
+
+        self.run_dbt()
+
+        self.assertTablesEqual("seed", "view")
+        self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")

test/integration/021_concurrency_test/update.sql

@@ -0,0 +1,101 @@
+insert into "concurrency_021"."seed" (first_name, last_name, email, gender, ip_address) values
+('Michael', 'Perez', '[email protected]', 'Male', '106.239.70.175'),
+('Shawn', 'Mccoy', '[email protected]', 'Male', '24.165.76.182'),
+('Kathleen', 'Payne', '[email protected]', 'Female', '113.207.168.106'),
+('Jimmy', 'Cooper', '[email protected]', 'Male', '198.24.63.114'),
+('Katherine', 'Rice', '[email protected]', 'Female', '36.97.186.238'),
+('Sarah', 'Ryan', '[email protected]', 'Female', '119.117.152.40'),
+('Martin', 'Mcdonald', '[email protected]', 'Male', '8.76.38.115'),
+('Frank', 'Robinson', '[email protected]', 'Male', '186.14.64.194'),
+('Jennifer', 'Franklin', '[email protected]', 'Female', '91.216.3.131'),
+('Henry', 'Welch', '[email protected]', 'Male', '176.35.182.168'),
+('Fred', 'Snyder', '[email protected]', 'Male', '217.106.196.54'),
+('Amy', 'Dunn', '[email protected]', 'Female', '95.39.163.195'),
+('Kathleen', 'Meyer', '[email protected]', 'Female', '164.142.188.214'),
+('Steve', 'Ferguson', '[email protected]', 'Male', '138.22.204.251'),
+('Teresa', 'Hill', '[email protected]', 'Female', '82.84.228.235'),
+('Amanda', 'Harper', '[email protected]', 'Female', '16.123.56.176'),
+('Kimberly', 'Ray', '[email protected]', 'Female', '48.66.48.12'),
+('Johnny', 'Knight', '[email protected]', 'Male', '99.30.138.123'),
+('Virginia', 'Freeman', '[email protected]', 'Female', '225.172.182.63'),
+('Anna', 'Austin', '[email protected]', 'Female', '62.111.227.148'),
+('Willie', 'Hill', '[email protected]', 'Male', '0.86.232.249'),
+('Sean', 'Harris', '[email protected]', 'Male', '117.165.133.249'),
+('Mildred', 'Adams', '[email protected]', 'Female', '163.44.97.46'),
+('David', 'Graham', '[email protected]', 'Male', '78.13.246.202'),
+('Victor', 'Hunter', '[email protected]', 'Male', '64.156.179.139'),
+('Aaron', 'Ruiz', '[email protected]', 'Male', '34.194.68.78'),
+('Benjamin', 'Brooks', '[email protected]', 'Male', '20.192.189.107'),
+('Lisa', 'Wilson', '[email protected]', 'Female', '199.152.130.217'),
+('Benjamin', 'King', '[email protected]', 'Male', '29.189.189.213'),
+('Christina', 'Williamson', '[email protected]', 'Female', '194.101.52.60'),
+('Jane', 'Gonzalez', '[email protected]', 'Female', '109.119.12.87'),
+('Thomas', 'Owens', '[email protected]', 'Male', '84.168.213.153'),
+('Katherine', 'Moore', '[email protected]', 'Female', '183.150.65.24'),
+('Jennifer', 'Stewart', '[email protected]', 'Female', '38.41.244.58'),
+('Sara', 'Tucker', '[email protected]', 'Female', '181.130.59.184'),
+('Harold', 'Ortiz', '[email protected]', 'Male', '198.231.63.137'),
+('Shirley', 'James', '[email protected]', 'Female', '83.27.160.104'),
+('Dennis', 'Johnson', '[email protected]', 'Male', '183.178.246.101'),
+('Louise', 'Weaver', '[email protected]', 'Female', '1.14.110.18'),
+('Maria', 'Armstrong', '[email protected]', 'Female', '181.142.1.249'),
+('Gloria', 'Cruz', '[email protected]', 'Female', '178.232.140.243'),
+('Diana', 'Spencer', '[email protected]', 'Female', '125.153.138.244'),
+('Kelly', 'Nguyen', '[email protected]', 'Female', '170.13.201.119'),
+('Jane', 'Rodriguez', '[email protected]', 'Female', '12.102.249.81'),
+('Scott', 'Brown', '[email protected]', 'Male', '108.174.99.192'),
+('Norma', 'Cruz', '[email protected]', 'Female', '201.112.156.197'),
+('Marie', 'Peters', '[email protected]', 'Female', '231.121.197.144'),
+('Lillian', 'Carr', '[email protected]', 'Female', '206.179.164.163'),
+('Judy', 'Nichols', '[email protected]', 'Female', '158.190.209.194'),
+('Billy', 'Long', '[email protected]', 'Male', '175.20.23.160'),
+('Howard', 'Reid', '[email protected]', 'Male', '118.99.196.20'),
+('Laura', 'Ferguson', '[email protected]', 'Female', '22.77.87.110'),
+('Anne', 'Bailey', '[email protected]', 'Female', '58.144.159.245'),
+('Rose', 'Morgan', '[email protected]', 'Female', '118.127.97.4'),
+('Nicholas', 'Reyes', '[email protected]', 'Male', '50.135.10.252'),
+('Joshua', 'Kennedy', '[email protected]', 'Male', '154.6.163.209'),
+('Paul', 'Watkins', '[email protected]', 'Male', '177.236.120.87'),
+('Kathryn', 'Kelly', '[email protected]', 'Female', '70.28.61.86'),
+('Adam', 'Armstrong', '[email protected]', 'Male', '133.235.24.202'),
+('Norma', 'Wallace', '[email protected]', 'Female', '241.119.227.128'),
+('Timothy', 'Reyes', '[email protected]', 'Male', '86.28.23.26'),
+('Elizabeth', 'Patterson', '[email protected]', 'Female', '139.97.159.149'),
+('Edward', 'Gomez', '[email protected]', 'Male', '158.103.108.255'),
+('David', 'Cox', '[email protected]', 'Male', '206.80.80.58'),
+('Brenda', 'Wood', '[email protected]', 'Female', '217.207.44.179'),
+('Adam', 'Walker', '[email protected]', 'Male', '253.211.54.93'),
+('Michael', 'Hart', '[email protected]', 'Male', '230.206.200.22'),
+('Jesse', 'Ellis', '[email protected]', 'Male', '213.254.162.52'),
+('Janet', 'Powell', '[email protected]', 'Female', '27.192.194.86'),
+('Helen', 'Ford', '[email protected]', 'Female', '52.160.102.168'),
+('Gerald', 'Carpenter', '[email protected]', 'Male', '36.30.194.218'),
+('Kathryn', 'Oliver', '[email protected]', 'Female', '202.63.103.69'),
+('Alan', 'Berry', '[email protected]', 'Male', '246.157.112.211'),
+('Harry', 'Andrews', '[email protected]', 'Male', '195.108.0.12'),
+('Andrea', 'Hall', '[email protected]', 'Female', '149.162.163.28'),
+('Barbara', 'Wells', '[email protected]', 'Female', '224.70.72.1'),
+('Anne', 'Wells', '[email protected]', 'Female', '180.168.81.153'),
+('Harry', 'Harper', '[email protected]', 'Male', '151.87.130.21'),
+('Jack', 'Ray', '[email protected]', 'Male', '220.109.38.178'),
+('Phillip', 'Hamilton', '[email protected]', 'Male', '166.40.47.30'),
+('Shirley', 'Hunter', '[email protected]', 'Female', '97.209.140.194'),
+('Arthur', 'Daniels', '[email protected]', 'Male', '5.40.240.86'),
+('Virginia', 'Rodriguez', '[email protected]', 'Female', '96.80.164.184'),
+('Christina', 'Ryan', '[email protected]', 'Female', '56.35.5.52'),
+('Theresa', 'Mendoza', '[email protected]', 'Female', '243.42.0.210'),
+('Jason', 'Cole', '[email protected]', 'Male', '198.248.39.129'),
+('Phillip', 'Bryant', '[email protected]', 'Male', '140.39.116.251'),
+('Adam', 'Torres', '[email protected]', 'Male', '101.75.187.135'),
+('Margaret', 'Johnston', '[email protected]', 'Female', '159.30.69.149'),
+('Paul', 'Payne', '[email protected]', 'Male', '199.234.140.220'),
+('Todd', 'Willis', '[email protected]', 'Male', '191.59.136.214'),
+('Willie', 'Oliver', '[email protected]', 'Male', '44.212.35.197'),
+('Frances', 'Robertson', '[email protected]', 'Female', '31.117.65.136'),
+('Gregory', 'Hawkins', '[email protected]', 'Male', '91.3.22.49'),
+('Lisa', 'Perkins', '[email protected]', 'Female', '145.95.31.186'),
+('Jacqueline', 'Anderson', '[email protected]', 'Female', '14.176.0.187'),
+('Shirley', 'Diaz', '[email protected]', 'Female', '207.12.95.46'),
+('Nicole', 'Meyer', '[email protected]', 'Female', '231.79.115.13'),
+('Mary', 'Gray', '[email protected]', 'Female', '210.116.64.253'),
+('Jean', 'Mcdonald', '[email protected]', 'Female', '122.239.235.117');