Skip to content

Commit

Permalink
[AIRFLOW-4472] Use json.dumps/loads for templating lineage data (#5253)
Browse files Browse the repository at this point in the history
jinja2 cannot use dict/lists as templates hence converting
it to json solves this while keeping complexity down.

(cherry picked from commit a6daeb5)
  • Loading branch information
bolkedebruin authored and kaxil committed Jun 19, 2020
1 parent 2bdf471 commit ba645ec
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 4 deletions.
11 changes: 9 additions & 2 deletions airflow/lineage/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
import six

from typing import List
Expand Down Expand Up @@ -62,7 +63,11 @@ def __getattr__(self, attr):
if attr in self.attributes:
if self.context:
env = Environment()
return env.from_string(self._data.get(attr)).render(**self.context)
# dump to json here in order to be able to manage dicts and lists
rendered = env.from_string(
json.dumps(self._data.get(attr))
).render(**self.context)
return json.loads(rendered)

return self._data.get(attr)

Expand All @@ -82,7 +87,9 @@ def as_dict(self):
env = Environment()
if self.context:
for key, value in six.iteritems(attributes):
attributes[key] = env.from_string(value).render(**self.context)
attributes[key] = json.loads(
env.from_string(json.dumps(value)).render(**self.context)
)

d = {
"typeName": self.type_name,
Expand Down
4 changes: 2 additions & 2 deletions airflow/models/baseoperator.py
Original file line number Diff line number Diff line change
Expand Up @@ -428,8 +428,8 @@ def __init__(
self._log = logging.getLogger("airflow.task.operators")

# lineage
self.inlets = [] # type: Iterable[DataSet]
self.outlets = [] # type: Iterable[DataSet]
self.inlets = [] # type: List[DataSet]
self.outlets = [] # type: List[DataSet]
self.lineage_data = None

self._inlets = {
Expand Down

0 comments on commit ba645ec

Please sign in to comment.