add macros to flat graph

dbt/contracts/graph/compiled.py

@@ -1,5 +1,4 @@
-from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \
-    Length
+from voluptuous import Schema, Required, All, Any, Length
 
 from collections import OrderedDict
 
@@ -8,10 +7,11 @@
 from dbt.logger import GLOBAL_LOGGER as logger
 
 from dbt.contracts.common import validate_with
-from dbt.contracts.graph.parsed import parsed_graph_item_contract
+from dbt.contracts.graph.parsed import parsed_node_contract, \
+    parsed_macro_contract
 
 
-compiled_graph_item_contract = parsed_graph_item_contract.extend({
+compiled_node_contract = parsed_node_contract.extend({
     # compiled fields
     Required('compiled'): bool,
     Required('compiled_sql'): Any(basestring, None),
@@ -24,16 +24,25 @@
     Required('injected_sql'): Any(basestring, None),
 })
 
+compiled_nodes_contract = Schema({
+    str: compiled_node_contract,
+})
 
-def validate_one(compiled_graph_item):
-    validate_with(compiled_graph_item_contract, compiled_graph_item)
+compiled_macro_contract = parsed_macro_contract
 
+compiled_macros_contract = Schema({
+    str: compiled_macro_contract,
+})
 
-def validate(compiled_graph):
-    for k, v in compiled_graph.items():
-        validate_with(compiled_graph_item_contract, v)
+compiled_graph_contract = Schema({
+    Required('nodes'): compiled_nodes_contract,
+    Required('macros'): compiled_macros_contract,
+})
 
-        if v.get('unique_id') != k:
-            error_msg = 'unique_id must match key name in compiled graph!'
-            logger.info(error_msg)
-            raise ValidationException(error_msg)
+
+def validate_node(compiled_node):
+    validate_with(compiled_node_contract, compiled_node)
+
+
+def validate(compiled_graph):
+    validate_with(compiled_graph_contract, compiled_graph)

dbt/contracts/graph/parsed.py

@@ -1,12 +1,15 @@
-from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \
-    Length
+from voluptuous import Schema, Required, All, Any, Length, Optional
+
+import jinja2.runtime
 
 from dbt.compat import basestring
-from dbt.exceptions import ValidationException
-from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.model import NodeType
 
 from dbt.contracts.common import validate_with
-from dbt.contracts.graph.unparsed import unparsed_graph_item_contract
+from dbt.contracts.graph.unparsed import unparsed_node_contract, \
+    unparsed_base_contract
+
+from dbt.logger import GLOBAL_LOGGER as logger # noqa
 
 
 config_contract = {
@@ -25,40 +28,54 @@
     Optional('dist'): basestring,
 }
 
-
-parsed_graph_item_contract = unparsed_graph_item_contract.extend({
+parsed_node_contract = unparsed_node_contract.extend({
     # identifiers
     Required('unique_id'): All(basestring, Length(min=1, max=255)),
     Required('fqn'): All(list, [All(basestring)]),
 
     # parsed fields
-    Required('depends_on'): All(list,
-                                [All(basestring, Length(min=1, max=255))]),
+    Required('depends_on'): {
+        Required('nodes'): [All(basestring, Length(min=1, max=255))],
+        Required('macros'): [All(basestring, Length(min=1, max=255))],
+    },
     Required('empty'): bool,
     Required('config'): config_contract,
     Required('tags'): All(set),
 })
 
+parsed_nodes_contract = Schema({
+    str: parsed_node_contract,
+})
+
+parsed_macro_contract = unparsed_base_contract.extend({
+    # identifiers
+    Required('resource_type'): Any(NodeType.Macro),
+    Required('unique_id'): All(basestring, Length(min=1, max=255)),
+    Required('tags'): All(set),
+
+    # contents
+    Required('parsed_macro'): jinja2.runtime.Macro
+
+})
+
+parsed_macros_contract = Schema({
+    str: parsed_macro_contract,
+})
+
+
+parsed_graph_contract = Schema({
+    Required('nodes'): parsed_nodes_contract,
+    Required('macros'): parsed_macros_contract,
+})
+
 
-def validate_one(parsed_graph_item):
-    validate_with(parsed_graph_item_contract, parsed_graph_item)
+def validate_nodes(parsed_nodes):
+    validate_with(parsed_nodes_contract, parsed_nodes)
 
-    materialization = parsed_graph_item.get('config', {}) \
-                                       .get('materialized')
 
-    if materialization == 'incremental' and \
-       parsed_graph_item.get('config', {}).get('sql_where') is None:
-        raise ValidationException(
-            'missing `sql_where` for an incremental model')
+def validate_macros(parsed_macros):
+    validate_with(parsed_macros_contract, parsed_macros)
 
 
 def validate(parsed_graph):
-    for k, v in parsed_graph.items():
-        validate_one(v)
-
-        if v.get('unique_id') != k:
-            error_msg = ('unique_id must match key name in parsed graph!'
-                         'key: {}, model: {}'
-                         .format(k, v))
-            logger.info(error_msg)
-            raise ValidationException(error_msg)
+    validate_with(parsed_graph_contract, parsed_graph)

dbt/contracts/graph/unparsed.py

@@ -1,27 +1,29 @@
-from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \
-    Length
+from voluptuous import Schema, Required, All, Any, Length
 
 from dbt.compat import basestring
 from dbt.contracts.common import validate_with
-from dbt.logger import GLOBAL_LOGGER as logger
 
 from dbt.model import NodeType
 
-unparsed_graph_item_contract = Schema({
+unparsed_base_contract = Schema({
     # identifiers
     Required('name'): All(basestring, Length(min=1, max=127)),
     Required('package_name'): basestring,
-    Required('resource_type'): Any(NodeType.Model,
-                                   NodeType.Test,
-                                   NodeType.Analysis),
 
     # filesystem
     Required('root_path'): basestring,
     Required('path'): basestring,
     Required('raw_sql'): basestring,
 })
 
+unparsed_node_contract = unparsed_base_contract.extend({
+    Required('resource_type'): Any(NodeType.Model,
+                                   NodeType.Test,
+                                   NodeType.Analysis)
+})
+
+unparsed_nodes_contract = Schema([unparsed_node_contract])
+
 
-def validate(unparsed_graph):
-    for item in unparsed_graph:
-        validate_with(unparsed_graph_item_contract, item)
+def validate_nodes(nodes):
+    validate_with(unparsed_nodes_contract, nodes)

dbt/model.py

@@ -20,6 +20,7 @@ class NodeType(object):
     Analysis = 'analysis'
     Test = 'test'
     Archive = 'archive'
+    Macro = 'macro'
 
 
 class SourceConfig(object):