add parser

dbt/adapters/snowflake.py

@@ -182,7 +182,7 @@ def rename(cls, profile, from_name, to_name, model_name=None):
 
     @classmethod
     def execute_model(cls, profile, model):
-        parts = re.split(r'-- (DBT_OPERATION .*)', model.compiled_contents)
+        parts = re.split(r'-- (DBT_OPERATION .*)', model.get('wrapped_sql'))
         connection = cls.get_connection(profile)
 
         if flags.STRICT_MODE:
@@ -216,7 +216,7 @@ def call_expand_target_column_types(kwargs):
                 func_map[function](kwargs)
             else:
                 handle, cursor = cls.add_query_to_transaction(
-                    part, connection, model.name)
+                    part, connection, model.get('name'))
 
         handle.commit()
 

dbt/compat.py

@@ -0,0 +1,45 @@
+import codecs
+
+WHICH_PYTHON = None
+
+try:
+    basestring
+    WHICH_PYTHON = 2
+except NameError:
+    WHICH_PYTHON = 3
+
+if WHICH_PYTHON == 2:
+    basestring = basestring
+else:
+    basestring = str
+
+
+def to_unicode(s):
+    if WHICH_PYTHON == 2:
+        return unicode(s)
+    else:
+        return str(s)
+
+
+def to_string(s):
+    if WHICH_PYTHON == 2:
+        if isinstance(s, unicode):
+            return s
+        elif isinstance(s, basestring):
+            return to_unicode(s)
+        else:
+            return to_unicode(str(s))
+    else:
+        if isinstance(s, basestring):
+            return s
+        else:
+            return str(s)
+
+
+def write_file(path, s):
+    if WHICH_PYTHON == 2:
+        with codecs.open(path, 'w', encoding='utf-8') as f:
+            return f.write(to_string(s))
+    else:
+        with open(path, 'w') as f:
+            return f.write(to_string(s))

dbt/compilation.py

@@ -11,11 +11,12 @@
 from dbt.model import Model, NodeType
 from dbt.source import Source
 from dbt.utils import find_model_by_fqn, find_model_by_name, \
-     split_path, This, Var, compiler_error, to_string
+     split_path, This, Var, compiler_error
 
 from dbt.linker import Linker
 from dbt.runtime import RuntimeContext
 
+import dbt.compat
 import dbt.contracts.graph.compiled
 import dbt.contracts.graph.parsed
 import dbt.contracts.project
@@ -33,10 +34,27 @@
 graph_file_name = 'graph.yml'
 
 
+def compile_and_print_status(project, args):
+    compiler = Compiler(project, args)
+    compiler.initialize()
+    results = {
+        NodeType.Model: 0,
+        NodeType.Test: 0,
+        NodeType.Archive: 0,
+    }
+
+    results.update(compiler.compile())
+
+    stat_line = ", ".join(
+        ["{} {}s".format(ct, t) for t, ct in results.items()])
+
+    logger.info("Compiled {}".format(stat_line))
+
+
 def compile_string(string, ctx):
     try:
         env = jinja2.Environment()
-        template = env.from_string(str(string), globals=ctx)
+        template = env.from_string(dbt.compat.to_string(string), globals=ctx)
         return template.render(ctx)
     except jinja2.exceptions.TemplateSyntaxError as e:
         compiler_error(None, str(e))
@@ -130,7 +148,7 @@ def inject_ctes_into_sql(sql, ctes):
         with_stmt,
         sqlparse.sql.Token(sqlparse.tokens.Keyword, ", ".join(ctes)))
 
-    return str(parsed)
+    return dbt.compat.to_string(parsed)
 
 
 class Compiler(object):
@@ -158,8 +176,7 @@ def __write(self, build_filepath, payload):
         if not os.path.exists(os.path.dirname(target_path)):
             os.makedirs(os.path.dirname(target_path))
 
-        with open(target_path, 'w') as f:
-            f.write(to_string(payload))
+        dbt.compat.write_file(target_path, payload)
 
     def __model_config(self, model, linker):
         def do_config(*args, **kwargs):
@@ -224,7 +241,8 @@ def wrapped_do_ref(*args):
                 logger.info("Compiler error in {}".format(model.get('path')))
                 logger.info("Enabled models:")
                 for n, m in all_models.items():
-                    logger.info(" - {}".format(".".join(m.get('fqn'))))
+                    if m.get('resource_type') == NodeType.Model:
+                        logger.info(" - {}".format(m.get('unique_id')))
                 raise e
 
         return wrapped_do_ref
@@ -397,7 +415,8 @@ def compile_nodes(self, linker, nodes, macro_generator):
                     injected_node.get('path'),
                     all_projects.get(injected_node.get('package_name')))
 
-                model._config = injected_node.get('config', {})
+                cfg = injected_node.get('config', {})
+                model._config = cfg
 
                 context = self.get_context(linker, model, injected_nodes)
 

dbt/contracts/connection.py

@@ -1,5 +1,6 @@
 from voluptuous import Schema, Required, All, Any, Extra, Range, Optional
 
+from dbt.compat import basestring
 from dbt.contracts.common import validate_with
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -12,22 +13,22 @@
 })
 
 postgres_credentials_contract = Schema({
-    Required('dbname'): str,
-    Required('host'): str,
-    Required('user'): str,
-    Required('pass'): str,
+    Required('dbname'): basestring,
+    Required('host'): basestring,
+    Required('user'): basestring,
+    Required('pass'): basestring,
     Required('port'): All(int, Range(min=0, max=65535)),
-    Required('schema'): str,
+    Required('schema'): basestring,
 })
 
 snowflake_credentials_contract = Schema({
-    Required('account'): str,
-    Required('user'): str,
-    Required('password'): str,
-    Required('database'): str,
-    Required('schema'): str,
-    Required('warehouse'): str,
-    Optional('role'): str,
+    Required('account'): basestring,
+    Required('user'): basestring,
+    Required('password'): basestring,
+    Required('database'): basestring,
+    Required('schema'): basestring,
+    Required('warehouse'): basestring,
+    Optional('role'): basestring,
 })
 
 credentials_mapping = {

dbt/contracts/graph/compiled.py

@@ -1,6 +1,7 @@
 from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \
     Length
 
+from dbt.compat import basestring
 from dbt.exceptions import ValidationException
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -11,13 +12,13 @@
 compiled_graph_item_contract = parsed_graph_item_contract.extend({
     # compiled fields
     Required('compiled'): bool,
-    Required('compiled_sql'): Any(str, None),
+    Required('compiled_sql'): Any(basestring, None),
 
     # injected fields
     Required('extra_ctes_injected'): bool,
-    Required('extra_cte_ids'): All(list, [str]),
-    Required('extra_cte_sql'): All(list, [str]),
-    Required('injected_sql'): Any(str, None),
+    Required('extra_cte_ids'): All(list, [basestring]),
+    Required('extra_cte_sql'): All(list, [basestring]),
+    Required('injected_sql'): Any(basestring, None),
 })
 
 

dbt/contracts/graph/parsed.py

@@ -1,6 +1,7 @@
 from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \
     Length
 
+from dbt.compat import basestring
 from dbt.exceptions import ValidationException
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -16,25 +17,26 @@
     Required('vars'): dict,
 
     # incremental optional fields
-    Optional('sql_where'): str,
-    Optional('unique_key'): str,
+    Optional('sql_where'): basestring,
+    Optional('unique_key'): basestring,
 
     # adapter optional fields
-    Optional('sort'): str,
-    Optional('dist'): str,
+    Optional('sort'): basestring,
+    Optional('dist'): basestring,
 }
 
 
 parsed_graph_item_contract = unparsed_graph_item_contract.extend({
     # identifiers
-    Required('unique_id'): All(str, Length(min=1, max=255)),
-    Required('fqn'): All(list, [All(str)]),
+    Required('unique_id'): All(basestring, Length(min=1, max=255)),
+    Required('fqn'): All(list, [All(basestring)]),
 
     # parsed fields
-    Required('depends_on'): All(list, [All(str, Length(min=1, max=255))]),
+    Required('depends_on'): All(list,
+                                [All(basestring, Length(min=1, max=255))]),
     Required('empty'): bool,
     Required('config'): config_contract,
-    Required('tags'): All(list, [str]),
+    Required('tags'): All(list, [basestring]),
 })
 
 

dbt/contracts/graph/unparsed.py

@@ -1,21 +1,22 @@
 from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \
     Length
 
+from dbt.compat import basestring
 from dbt.contracts.common import validate_with
 from dbt.logger import GLOBAL_LOGGER as logger
 
 from dbt.model import NodeType
 
 unparsed_graph_item_contract = Schema({
     # identifiers
-    Required('name'): All(str, Length(min=1, max=63)),
-    Required('package_name'): str,
+    Required('name'): All(basestring, Length(min=1, max=63)),
+    Required('package_name'): basestring,
     Required('resource_type'): Any(NodeType.Model, NodeType.Test),
 
     # filesystem
-    Required('root_path'): str,
-    Required('path'): str,
-    Required('raw_sql'): str,
+    Required('root_path'): basestring,
+    Required('path'): basestring,
+    Required('raw_sql'): basestring,
 })
 
 

dbt/main.py

@@ -28,7 +28,7 @@ def main(args=None):
         args = sys.argv[1:]
 
     try:
-        return handle(args)
+        handle(args)
 
     except RuntimeError as e:
         logger.info("Encountered an error:")

dbt/model.py

@@ -2,6 +2,8 @@
 import yaml
 import jinja2
 import re
+
+from dbt.compat import basestring
 from dbt.templates import BaseCreateTemplate, ArchiveInsertTemplate
 from dbt.utils import split_path
 import dbt.project
@@ -396,12 +398,6 @@ def build_path(self):
         return os.path.join(*path_parts)
 
     def compile_string(self, ctx, string):
-        # python 2+3 check for stringiness
-        try:
-            basestring
-        except NameError:
-            basestring = str
-
         # if bool/int/float/etc are passed in, don't compile anything
         if not isinstance(string, basestring):
             return string

dbt/parser.py

@@ -123,7 +123,7 @@ def get_fqn(path, package_project_config, extra=[]):
     parts = dbt.utils.split_path(path)
     name, _ = os.path.splitext(parts[-1])
     fqn = ([package_project_config.get('name')] +
-           parts[1:-1] +
+           parts[:-1] +
            extra +
            [name])