From 402d9d4562e0a1e00ea04cc731a2f39d7a3d1f24 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 14:56:31 +0100 Subject: [PATCH 01/19] Allow easier string formatting of queries, with optional descriptions of query attributes included in the resulting string --- flowmachine/flowmachine/core/query.py | 41 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/flowmachine/flowmachine/core/query.py b/flowmachine/flowmachine/core/query.py index 4bbc8f8442..33afc74630 100644 --- a/flowmachine/flowmachine/core/query.py +++ b/flowmachine/flowmachine/core/query.py @@ -112,16 +112,51 @@ def md5(self): self._md5 = md5(str(hashes).encode()).hexdigest() return self._md5 + query_id = md5 # alias which is more meaningful to users than 'md5' + @abstractmethod def _make_query(self): raise NotImplementedError def __repr__(self): + # Default representation, derived classes might want to add something more specific + return format(self, "query_id") + + def __format__(self, fmt=""): + """ + Return a formatted string representation of this query object. + + Parameters + ---------- + fmt : str, optional + This should be the empty string or a comma-separated list of + query attributes that will be included in the formatted string. + + Examples + -------- + + >>> dl = daily_location(date="2016-01-01", method="last") + >>> format(dl) + + >>> format(dl, "query_id,is_stored") + + >>> print(f"{dl:is_stored,query_state}") + + """ + query_descr = f"Query of type: {self.__class__.__name__}" + attrs_to_include = [] if fmt == "" else fmt.split(",") + attr_descriptions = [] + for attr in attrs_to_include: + try: + attr_descriptions.append(f"{attr}: {getattr(self, attr)!r}") + except AttributeError: + raise ValueError( + f"Format string contains invalid query attribute: '{attr}'" + ) - # Default representation, derived classes might want to - # add something more specific - return f"" + all_descriptions = [query_descr] + attr_descriptions + return f"<{', '.join(all_descriptions)}>" def __iter__(self): con = self.connection.engine From df23a7be236ab7ab28673f6194ad92a3b671c32c Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 15:00:28 +0100 Subject: [PATCH 02/19] Optionally display info whether queries in dependency tree are stored --- flowmachine/flowmachine/utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/flowmachine/flowmachine/utils.py b/flowmachine/flowmachine/utils.py index f9c6459506..c0bbc289a7 100644 --- a/flowmachine/flowmachine/utils.py +++ b/flowmachine/flowmachine/utils.py @@ -379,7 +379,7 @@ def sort_recursively(d): return d -def print_dependency_tree(query_obj, stream=None, indent_level=0): +def print_dependency_tree(query_obj, show_stored=False, stream=None, indent_level=0): """ Print the dependencies of a flowmachine query in a tree-like structure. @@ -387,6 +387,8 @@ def print_dependency_tree(query_obj, stream=None, indent_level=0): ---------- query_obj : Query An instance of a query object. + show_stored : bool, optional + If True, show for each query whether it is stored or not. Default: False. stream : io.IOBase, optional The stream to which the output should be written (default: stdout). indent_level : int @@ -398,10 +400,13 @@ def print_dependency_tree(query_obj, stream=None, indent_level=0): indent_per_level = 3 indent = " " * (indent_per_level * indent_level - 1) prefix = "" if indent_level == 0 else "- " - stream.write(f"{indent}{prefix}{query_obj}\n") + fmt = "query_id" if not show_stored else "query_id,is_stored" + stream.write(f"{indent}{prefix}{query_obj:{fmt}}\n") deps_sorted_by_query_id = sorted(query_obj.dependencies, key=lambda q: q.md5) for dep in deps_sorted_by_query_id: - print_dependency_tree(dep, indent_level=indent_level + 1, stream=stream) + print_dependency_tree( + dep, indent_level=indent_level + 1, stream=stream, show_stored=show_stored + ) def _get_query_attrs_for_dependency_graph(query_obj, analyse=False): From c0d5ee3dba9744bab9f22c98d25a179d827b277c Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 15:20:37 +0100 Subject: [PATCH 03/19] Tweak docstring and node color for stored queries --- flowmachine/flowmachine/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flowmachine/flowmachine/utils.py b/flowmachine/flowmachine/utils.py index c0bbc289a7..323d4c0be6 100644 --- a/flowmachine/flowmachine/utils.py +++ b/flowmachine/flowmachine/utils.py @@ -444,6 +444,8 @@ def calculate_dependency_graph(query_obj, analyse=False): as node attributes. The resulting networkx object can then be visualised, or analysed. + When visualised, nodes corresponding to stored queries will be + rendered green. The dependency graph includes the estimated cost of the query in the 'cost' attribute, the query object the node represents in the 'query_object' attribute, and with the analyse @@ -467,9 +469,11 @@ def calculate_dependency_graph(query_obj, analyse=False): export it to an SVG string and display it directly in the notebook: >>> import flowmachine + >>> import networkx as nx >>> from flowmachine.features import daily_location >>> from flowmachine.utils import calculate_dependency_graph >>> from io import BytesIO + >>> from IPython.display import SVG >>> flowmachine.connect(flowdb_user="flowdb", flowdb_password="flowflow", redis_password="fm_redis") >>> dl = daily_location(date="2016-01-01") >>> G = calculate_dependency_graph(dl, analyse=True) @@ -518,12 +522,12 @@ def calculate_dependency_graph(query_obj, analyse=False): for n in set(y): attrs = _get_query_attrs_for_dependency_graph(n, analyse=analyse) attrs["shape"] = "rect" - attrs["label"] = "{}. Cost: {}.".format(attrs["name"], attrs["cost"]) attrs["query_object"] = n + attrs["label"] = f"{attrs['name']}. Cost: {attrs['cost']}" if analyse: attrs["label"] += " Actual runtime: {}.".format(attrs["runtime"]) if attrs["stored"]: - attrs["fillcolor"] = "green" + attrs["fillcolor"] = "#b3de69" # light green attrs["style"] = "filled" g.add_node(f"x{n.md5}", **attrs) From eb78051d14cacd20462be8b892e651747e0b1756 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 15:41:28 +0100 Subject: [PATCH 04/19] Add helper function to plot dependency graph directly --- flowmachine/flowmachine/utils.py | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/flowmachine/flowmachine/utils.py b/flowmachine/flowmachine/utils.py index 323d4c0be6..485cab6c24 100644 --- a/flowmachine/flowmachine/utils.py +++ b/flowmachine/flowmachine/utils.py @@ -11,6 +11,7 @@ import networkx as nx import structlog import sys +from io import BytesIO from pathlib import Path from pglast import prettify from psycopg2._psycopg import adapt @@ -536,3 +537,54 @@ def calculate_dependency_graph(query_obj, analyse=False): g.add_edge(*[f"x{z.md5}" for z in (x, y)]) return g + + +def plot_dependency_graph( + query_obj, analyse=False, format="png", width=None, height=None +): + """ + Plot a graph of all the queries that go into producing this one (see `calculate_dependency_graph` + for more details). This returns an IPython.display object which can be directly displayed in + Jupyter notebooks. + + Parameters + ---------- + query_obj : Query + Query object to plot a dependency graph for. + analyse : bool + Set to True to get actual runtimes for queries. Note that this will actually run the query! + format : {"png", "svg"} + Output format of the resulting + width : int + Width in pixels to which to constrain the image. Note this is only supported for format="png". + height : int + Height in pixels to which to constrain the image. Note this is only supported for format="png". + + Returns + ------- + IPython.display.Image or IPython.display.SVG + """ + try: + from IPython.display import Image, SVG + except ImportError: + raise ImportError( + "IPython is needed to plot dependency graph. Please install it." + ) + + G = calculate_dependency_graph(query_obj, analyse=analyse) + A = nx.nx_agraph.to_agraph(G) + s = BytesIO() + A.draw(s, format=format, prog="dot") + + if format == "png": + result = Image(s.getvalue(), width=width, height=height) + elif format == "svg": + if width is not None or height is not None: + logger.warning( + "The arguments 'width' and 'height' are not supported with format='svg'." + ) + result = SVG(s.getvalue().decode("utf8")) + else: + raise ValueError(f"Unsupported output format: '{format}'") + + return result From 1c524cf8c4b738ef7f700ccdd1f9fb0db2ef746f Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 15:41:48 +0100 Subject: [PATCH 05/19] Update docstring for calculate_dependency_graph() --- flowmachine/flowmachine/utils.py | 33 ++++++++------------------------ 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/flowmachine/flowmachine/utils.py b/flowmachine/flowmachine/utils.py index 485cab6c24..e5d0834c82 100644 --- a/flowmachine/flowmachine/utils.py +++ b/flowmachine/flowmachine/utils.py @@ -440,13 +440,13 @@ def _get_query_attrs_for_dependency_graph(query_obj, analyse=False): def calculate_dependency_graph(query_obj, analyse=False): """ - Produce a graph of all the queries that go into producing this - one, with their estimated run costs, and whether they are stored - as node attributes. + Produce a graph of all the queries that go into producing this one, with their estimated + run costs, and whether they are stored as node attributes. - The resulting networkx object can then be visualised, or analysed. - When visualised, nodes corresponding to stored queries will be - rendered green. + The resulting networkx object can then be visualised, or analysed. When visualised, + nodes corresponding to stored queries will be rendered green. See the function + `plot_dependency_graph()` for a convenient way of plotting a dependency graph directly + for visualisation in a Jupyter notebook. The dependency graph includes the estimated cost of the query in the 'cost' attribute, the query object the node represents in the 'query_object' attribute, and with the analyse @@ -466,25 +466,8 @@ def calculate_dependency_graph(query_obj, analyse=False): Examples -------- - A useful way to visualise the dependency graph in a Jupyter notebook is to - export it to an SVG string and display it directly in the notebook: - - >>> import flowmachine - >>> import networkx as nx - >>> from flowmachine.features import daily_location - >>> from flowmachine.utils import calculate_dependency_graph - >>> from io import BytesIO - >>> from IPython.display import SVG - >>> flowmachine.connect(flowdb_user="flowdb", flowdb_password="flowflow", redis_password="fm_redis") - >>> dl = daily_location(date="2016-01-01") - >>> G = calculate_dependency_graph(dl, analyse=True) - >>> A = nx.nx_agraph.to_agraph(G) - >>> svg_str = BytesIO() - >>> A.draw(svg_str, format="svg", prog="dot") - >>> svg_str = svg_str.getvalue().decode("utf8") - >>> SVG(svg_str) # within a Jupyter notebook this will be displayed as a graph - - Alternatively, you can export the dependency graph to a .dot file as follows: + If you don't want to visualise the dependency graph directly (for example + using `plot_dependency_graph()`, you can export it to a .dot file as follows: >>> import flowmachine >>> from flowmachine.features import daily_location From 354ebb0f22b044fb3181eabc66c56247fde88fab Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:00:35 +0100 Subject: [PATCH 06/19] Slightly terser import error message --- flowmachine/flowmachine/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flowmachine/flowmachine/utils.py b/flowmachine/flowmachine/utils.py index e5d0834c82..482d15c52b 100644 --- a/flowmachine/flowmachine/utils.py +++ b/flowmachine/flowmachine/utils.py @@ -550,9 +550,7 @@ def plot_dependency_graph( try: from IPython.display import Image, SVG except ImportError: - raise ImportError( - "IPython is needed to plot dependency graph. Please install it." - ) + raise ImportError("requires IPython ", "https://ipython.org/") G = calculate_dependency_graph(query_obj, analyse=analyse) A = nx.nx_agraph.to_agraph(G) From f35239d20f59bdc982d6c00ab7dc47f3b9f2e8a2 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:02:49 +0100 Subject: [PATCH 07/19] Add info about required packages --- flowmachine/flowmachine/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flowmachine/flowmachine/utils.py b/flowmachine/flowmachine/utils.py index 482d15c52b..0ddcdce5a1 100644 --- a/flowmachine/flowmachine/utils.py +++ b/flowmachine/flowmachine/utils.py @@ -530,6 +530,8 @@ def plot_dependency_graph( for more details). This returns an IPython.display object which can be directly displayed in Jupyter notebooks. + Note that this requires the IPython and pygraphviz packages to be installed. + Parameters ---------- query_obj : Query From 936b4e37f9d332f9f64fc35890b19e7020f1068e Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:03:08 +0100 Subject: [PATCH 08/19] Install pygraphviz as dev dependency --- flowmachine/Pipfile | 1 + flowmachine/Pipfile.lock | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/flowmachine/Pipfile b/flowmachine/Pipfile index 22ad8e4d60..c24b80fb5b 100644 --- a/flowmachine/Pipfile +++ b/flowmachine/Pipfile @@ -40,6 +40,7 @@ cachey = "*" approvaltests = "*" watchdog = "*" ipdb = "*" +pygraphviz = "*" [requires] python_version = "3.7" diff --git a/flowmachine/Pipfile.lock b/flowmachine/Pipfile.lock index 2bb3961113..695b4a31cd 100644 --- a/flowmachine/Pipfile.lock +++ b/flowmachine/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "365790cee03b5af858bc16ff80f80162d58f4e00a06d4b83e82cedfeeed3eeea" + "sha256": "e0d1144bda8ebed7d39298ed9ba14c04e1822a670a6a8be4121be7c704bf4712" }, "pipfile-spec": 6, "requires": { @@ -308,6 +308,14 @@ ], "version": "==1.4.3" }, + "appnope": { + "hashes": [ + "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", + "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.0" + }, "approvaltests": { "hashes": [ "sha256:15c18543f68dcdb124295b9fc0135096f86a9d973381ff5936e40a4457c02038" @@ -762,6 +770,13 @@ ], "version": "==2.4.0" }, + "pygraphviz": { + "hashes": [ + "sha256:50a829a305dc5a0fd1f9590748b19fece756093b581ac91e00c2c27c651d319d" + ], + "index": "pypi", + "version": "==1.5" + }, "pyparsing": { "hashes": [ "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", From f716666746ed10cd0ed7e095896f936b008b1d1a Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:14:47 +0100 Subject: [PATCH 09/19] Add simple test for test_plot_dependency_graph --- flowmachine/tests/test_utils.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/flowmachine/tests/test_utils.py b/flowmachine/tests/test_utils.py index 09953341f6..fed87425ef 100644 --- a/flowmachine/tests/test_utils.py +++ b/flowmachine/tests/test_utils.py @@ -11,6 +11,7 @@ import re import textwrap import unittest.mock +import IPython from io import StringIO from pathlib import Path @@ -27,6 +28,7 @@ _makesafe, print_dependency_tree, calculate_dependency_graph, + plot_dependency_graph, convert_dict_keys_to_strings, sort_recursively, time_period_add, @@ -252,9 +254,9 @@ def test_print_dependency_tree(): assert expected_output == output_with_query_ids_replaced -def test_dependency_graph(): +def test_calculate_dependency_graph(): """ - Test that dependency graph util runs and has some correct entries. + Test that calculate_dependency_graph() runs and the returned graph has some correct entries. """ query = daily_location("2016-01-01") G = calculate_dependency_graph(query, analyse=True) @@ -265,3 +267,17 @@ def test_dependency_graph(): ) assert f"x{sd.md5}" in G.nodes() assert G.nodes[f"x{sd.md5}"]["query_object"].md5 == sd.md5 + + +def test_plot_dependency_graph(): + """ + Test that plot_dependency_graph() runs and returns the expected IPython.display objects. + """ + query = daily_location(date="2016-01-02", level="admin2", method="most-common") + output_svg = plot_dependency_graph(query, format="svg") + output_png = plot_dependency_graph(query, format="png", width=600, height=200) + + assert isinstance(output_svg, IPython.display.SVG) + assert isinstance(output_png, IPython.display.Image) + assert output_png.width == 600 + assert output_png.height == 200 From 022a4949c249ddc2a895a063c89401a560b4d222 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:17:27 +0100 Subject: [PATCH 10/19] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 820efaf8f8..294314cb32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - FlowAPI now requires the `FLOWAPI_IDENTIFIER` environment variable to be set, which contains the name used to identify this FlowAPI server when generating tokens in FlowAuth [#727](https://github.com/Flowminder/FlowKit/issues/727) - `flowmachine.utils.calculate_dependency_graph` now includes the `Query` objects in the `query_object` field of the graph's nodes dictionary [#767](https://github.com/Flowminder/FlowKit/issues/767) - Architectural Decision Records (ADR) have been added and are included in the auto-generated docs [#780](https://github.com/Flowminder/FlowKit/issues/780) +- The function `print_dependency_tree()` now takes an optional argument `show_stored` to display information whether dependent queries have been stored or not [#804](https://github.com/Flowminder/FlowKit/issues/804) +- A new function `plot_dependency_graph()` has been added which allows to conveniently plot and visualise a dependency graph for use in Jupyter notebooks (this requires IPython and pygraphviz to be installed) [#786](https://github.com/Flowminder/FlowKit/issues/786) ### Changed From 39ec276bb92130f61511899e7e2a148baf79a678 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:26:42 +0100 Subject: [PATCH 11/19] Overwrite __format__ instead of __repr__ in Table to retain previous behaviour --- flowmachine/flowmachine/core/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowmachine/flowmachine/core/table.py b/flowmachine/flowmachine/core/table.py index 97f42ac29e..4b197032d2 100644 --- a/flowmachine/flowmachine/core/table.py +++ b/flowmachine/flowmachine/core/table.py @@ -124,7 +124,7 @@ def __init__(self, name=None, schema=None, columns=None): self._db_store_cache_metadata(compute_time=0) q_state_machine.finish() - def __repr__(self): + def __format__(self, fmt): return f"" @property From f5ff273fd71e74580066081013190a9c95db7265 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:29:30 +0100 Subject: [PATCH 12/19] Add no-cover pragma for warning message. --- flowmachine/flowmachine/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowmachine/flowmachine/utils.py b/flowmachine/flowmachine/utils.py index 0ddcdce5a1..66d154b2f9 100644 --- a/flowmachine/flowmachine/utils.py +++ b/flowmachine/flowmachine/utils.py @@ -562,7 +562,7 @@ def plot_dependency_graph( if format == "png": result = Image(s.getvalue(), width=width, height=height) elif format == "svg": - if width is not None or height is not None: + if width is not None or height is not None: # pragma: no cover logger.warning( "The arguments 'width' and 'height' are not supported with format='svg'." ) From b54fac1acf3899a19e345061986daa80f4cb8f5c Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 16:33:56 +0100 Subject: [PATCH 13/19] Tidy up imports; add test for to_nested_list() --- flowmachine/tests/test_utils.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/flowmachine/tests/test_utils.py b/flowmachine/tests/test_utils.py index fed87425ef..778d69a354 100644 --- a/flowmachine/tests/test_utils.py +++ b/flowmachine/tests/test_utils.py @@ -13,26 +13,11 @@ import unittest.mock import IPython from io import StringIO -from pathlib import Path from flowmachine.core import CustomQuery -from flowmachine.core.errors import BadLevelError from flowmachine.core.subscriber_subsetter import make_subscriber_subsetter from flowmachine.features import daily_location, EventTableSubset -from flowmachine.utils import ( - parse_datestring, - proj4string, - get_columns_for_level, - getsecret, - pretty_sql, - _makesafe, - print_dependency_tree, - calculate_dependency_graph, - plot_dependency_graph, - convert_dict_keys_to_strings, - sort_recursively, - time_period_add, -) +from flowmachine.utils import * @pytest.mark.parametrize("crs", (None, 4326, "+proj=longlat +datum=WGS84 +no_defs")) @@ -192,6 +177,15 @@ def test_convert_dict_keys_to_strings(): assert d_out_expected == d_out +def test_to_nested_list(): + """ + Test that a dictionary with multiple levels is correctly converted to a nested list of key-value pairs. + """ + d = {"a": {"b": 1, "c": [2, 3, {"e": 4}], "d": [5, 6]}} + expected = [("a", [("b", 1), ("c", [2, 3, [("e", 4)]]), ("d", [5, 6])])] + assert expected == to_nested_list(d) + + def test_sort_recursively(): """ Test that `sort_recursively` recursively sorts all components of the input dictionary. From b3a57cc841dcd7ce035deefcf296c82077c3da84 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 19:26:10 +0100 Subject: [PATCH 14/19] Need to explicitly import _makesafe (not covered by 'import *') --- flowmachine/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowmachine/tests/test_utils.py b/flowmachine/tests/test_utils.py index 778d69a354..b0829b34aa 100644 --- a/flowmachine/tests/test_utils.py +++ b/flowmachine/tests/test_utils.py @@ -5,7 +5,6 @@ """ Tests for flowmachine small helper functions """ -import datetime import pytest import pglast import re @@ -18,6 +17,7 @@ from flowmachine.core.subscriber_subsetter import make_subscriber_subsetter from flowmachine.features import daily_location, EventTableSubset from flowmachine.utils import * +from flowmachine.utils import _makesafe @pytest.mark.parametrize("crs", (None, 4326, "+proj=longlat +datum=WGS84 +no_defs")) From 13d9a9c94f992104c18e1fd089e772ca6c98de64 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 19:32:24 +0100 Subject: [PATCH 15/19] Install graphviz --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 20ec65cd9e..12aee9d10c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -290,6 +290,9 @@ jobs: - checkout - restore_cache: key: flowmachine-deps-1-{{ checksum "flowmachine/Pipfile.lock" }} + - run: + name: Install graphviz + command: sudo apt-get install -y xvfb graphviz - run: cd flowmachine && pipenv install --dev --deploy && pipenv run pip install -e . - save_cache: key: flowmachine-deps-1-{{ checksum "flowmachine/Pipfile.lock" }} From 2aa1f508fb3bba8f66c20f2eb4b9fc9ff1918c94 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 22:25:52 +0100 Subject: [PATCH 16/19] Swap order of steps --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12aee9d10c..9f53deda24 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -290,14 +290,14 @@ jobs: - checkout - restore_cache: key: flowmachine-deps-1-{{ checksum "flowmachine/Pipfile.lock" }} - - run: - name: Install graphviz - command: sudo apt-get install -y xvfb graphviz - run: cd flowmachine && pipenv install --dev --deploy && pipenv run pip install -e . - save_cache: key: flowmachine-deps-1-{{ checksum "flowmachine/Pipfile.lock" }} paths: - /home/circleci/.local/share/virtualenvs/flowmachine-caaCcVrN + - run: + name: Install graphviz + command: sudo apt-get install -y xvfb graphviz lint: docker: *base_docker From 7d43cca286872b12e10ff9c82f678a995f2b226f Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Thu, 16 May 2019 23:23:09 +0100 Subject: [PATCH 17/19] Move step into run_flowmachine_tests --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f53deda24..d5116010c2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -295,9 +295,6 @@ jobs: key: flowmachine-deps-1-{{ checksum "flowmachine/Pipfile.lock" }} paths: - /home/circleci/.local/share/virtualenvs/flowmachine-caaCcVrN - - run: - name: Install graphviz - command: sudo apt-get install -y xvfb graphviz lint: docker: *base_docker @@ -322,6 +319,9 @@ jobs: at: /home/circleci/ - restore_cache: key: flowmachine-deps-1-{{ checksum "Pipfile.lock" }} + - run: + name: Install graphviz + command: sudo apt-get install -y xvfb graphviz - run: *wait_for_flowdb - run: name: Run tests From 691bac8465f42082d509e96c23235b93fc3b23b2 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Fri, 17 May 2019 10:33:45 +0100 Subject: [PATCH 18/19] Add test for query formatting --- flowmachine/tests/test_query.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/flowmachine/tests/test_query.py b/flowmachine/tests/test_query.py index 64f8be7fc8..2054db3fac 100644 --- a/flowmachine/tests/test_query.py +++ b/flowmachine/tests/test_query.py @@ -135,3 +135,20 @@ def test_make_sql_no_overwrite(): dl = daily_location("2016-01-01") assert [] == dl._make_sql("admin3", schema="geography") + + +def test_query_formatting(): + """ + Test that query can be formatted as a string, with optional + """ + dl = daily_location("2016-01-01", method="last") + assert "" == format(dl) + assert ( + "" + == f"{dl:level,column_names}" + ) + + with pytest.raises( + ValueError, match="Format string contains invalid query attribute: 'foo'" + ): + format(dl, "query_id,foo") From 466184846af828686c8417c633a9cdfa33dff3d5 Mon Sep 17 00:00:00 2001 From: Maximilian Albert Date: Mon, 20 May 2019 10:41:10 +0100 Subject: [PATCH 19/19] Fix incomplete docstring --- flowmachine/tests/test_query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flowmachine/tests/test_query.py b/flowmachine/tests/test_query.py index 2054db3fac..b31686277e 100644 --- a/flowmachine/tests/test_query.py +++ b/flowmachine/tests/test_query.py @@ -139,7 +139,8 @@ def test_make_sql_no_overwrite(): def test_query_formatting(): """ - Test that query can be formatted as a string, with optional + Test that query can be formatted as a string, with query attributes + specified in the `fmt` argument being included. """ dl = daily_location("2016-01-01", method="last") assert "" == format(dl)