From e3ea7fd0aaddb7cac4849c4e9f80c332d0fa9852 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 12 Mar 2018 16:23:58 -0400 Subject: [PATCH 1/3] fix statements in on run start hooks --- dbt/node_runners.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index bc74b74558e..d52146ecde6 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -291,42 +291,47 @@ def run_hooks(cls, project, adapter, flat_graph, hook_type): nodes = flat_graph.get('nodes', {}).values() hooks = get_nodes_by_tags(nodes, {hook_type}, NodeType.Operation) - - # This will clear out an open transaction if there is one. - # on-run-* hooks should run outside of a transaction. This happens b/c - # psycopg2 automatically begins a transaction when a connection is - # created. TODO : Move transaction logic out of here, and implement - # a for-loop over these sql statements in jinja-land. Also, consider - # configuring psycopg2 (and other adapters?) to ensure that a - # transaction is only created if dbt initiates it. - conn_name = adapter.clear_transaction(profile) + adapter.clear_transaction(profile) compiled_hooks = [] for hook in hooks: + model_name = hook.get('name') + + # This will clear out an open transaction if there is one. + # on-run-* hooks should run outside of a transaction. This happens + # b/c psycopg2 automatically begins a transaction when a connection + # is created. TODO : Move transaction logic out of here, and + # implement a for-loop over these sql statements in jinja-land. + # Also, consider configuring psycopg2 (and other adapters?) to + # ensure that a transaction is only created if dbt initiates it. + adapter.clear_transaction(profile, model_name) compiled = cls._compile_node(adapter, project, hook, flat_graph) - model_name = compiled.get('name') statement = compiled['wrapped_sql'] hook_index = hook.get('index', len(hooks)) hook_dict = dbt.hooks.get_hook_dict(statement, index=hook_index) compiled_hooks.append(hook_dict) + adapter.release_connection(profile, model_name) ordered_hooks = sorted(compiled_hooks, key=lambda h: h.get('index', 0)) for hook in ordered_hooks: + model_name = compiled.get('name') + if dbt.flags.STRICT_MODE: dbt.contracts.graph.parsed.validate_hook(hook) sql = hook.get('sql', '') - adapter.execute_one(profile, sql, model_name=conn_name, + adapter.execute_one(profile, sql, model_name=model_name, auto_begin=False) - adapter.release_connection(profile, conn_name) + adapter.release_connection(profile, model_name) @classmethod def safe_run_hooks(cls, project, adapter, flat_graph, hook_type): try: cls.run_hooks(project, adapter, flat_graph, hook_type) - except dbt.exceptions.RuntimeException as e: + + except dbt.exceptions.RuntimeException: logger.info("Database error while running {}".format(hook_type)) raise From a9b6928869ac80e3a3e1748242f42c6076e4aa16 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 14 Mar 2018 11:46:00 -0400 Subject: [PATCH 2/3] do nothing on empty hook --- dbt/node_runners.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index d52146ecde6..491c1c48f8c 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -322,9 +322,12 @@ def run_hooks(cls, project, adapter, flat_graph, hook_type): dbt.contracts.graph.parsed.validate_hook(hook) sql = hook.get('sql', '') - adapter.execute_one(profile, sql, model_name=model_name, - auto_begin=False) - adapter.release_connection(profile, model_name) + + if len(sql) > 0: + adapter.execute_one(profile, sql, model_name=model_name, + auto_begin=False) + + adapter.release_connection(profile, model_name) @classmethod def safe_run_hooks(cls, project, adapter, flat_graph, hook_type): From c19b80fbbaab292efe306cc8936e660d34db590d Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 3 Apr 2018 14:17:03 -0400 Subject: [PATCH 3/3] pr feedback --- dbt/node_runners.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 491c1c48f8c..266c26cc837 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -14,7 +14,6 @@ import dbt.templates import dbt.writer -import os import time @@ -291,7 +290,6 @@ def run_hooks(cls, project, adapter, flat_graph, hook_type): nodes = flat_graph.get('nodes', {}).values() hooks = get_nodes_by_tags(nodes, {hook_type}, NodeType.Operation) - adapter.clear_transaction(profile) compiled_hooks = [] for hook in hooks: @@ -316,14 +314,14 @@ def run_hooks(cls, project, adapter, flat_graph, hook_type): ordered_hooks = sorted(compiled_hooks, key=lambda h: h.get('index', 0)) for hook in ordered_hooks: - model_name = compiled.get('name') + model_name = hook.get('name') if dbt.flags.STRICT_MODE: dbt.contracts.graph.parsed.validate_hook(hook) sql = hook.get('sql', '') - if len(sql) > 0: + if len(sql.strip()) > 0: adapter.execute_one(profile, sql, model_name=model_name, auto_begin=False)