From 4e38412d8240fbc74a13f93df07925d9ec8fecdf Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:15:45 +0100 Subject: [PATCH 01/48] Create vw_additional_wastewater_structure. Tackles https://github.com/teksi/wastewater/discussions/38 additional ws types are handled separately to not blow up the main ws view. Also, no extra-cols are allowed --- datamodel/app/view/create_views.py | 3 +- .../vw_additional_wastewater_structure.py | 579 ++++++++++++++++++ 2 files changed, 581 insertions(+), 1 deletion(-) create mode 100644 datamodel/app/view/vw_additional_wastewater_structure.py diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index f1fa992ae..dfd75b373 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -183,7 +183,8 @@ def create_views( srid, pg_service=pg_service, extra_definition=tww_wastewater_structure_extra ) vw_tww_reach(pg_service=pg_service, extra_definition=tww_reach_extra) - + vw_additional_wastewater_structure( + srid, pg_service=pg_service) run_sql("app/view/vw_file.sql", pg_service, variables) MultipleInheritance( diff --git a/datamodel/app/view/vw_additional_wastewater_structure.py b/datamodel/app/view/vw_additional_wastewater_structure.py new file mode 100644 index 000000000..64605c98a --- /dev/null +++ b/datamodel/app/view/vw_additional_wastewater_structure.py @@ -0,0 +1,579 @@ +#!/usr/bin/env python3 +# +# -- View: vw_additional_wastewater_structure + +import argparse +import os + +import psycopg2 +from pirogue.utils import insert_command, select_columns, table_parts, update_command +from yaml import safe_load + + +def vw_additional_wastewater_structure(srid: int, pg_service: str = None): + """ + Creates additional_wastewater_structure view + :param srid: EPSG code for geometries + :param pg_service: the PostgreSQL service name + """ + if not pg_service: + pg_service = os.getenv("PGSERVICE") + assert pg_service + + variables = {"SRID": int(srid)} + + conn = psycopg2.connect(f"service={pg_service}") + cursor = conn.cursor() + + view_sql = """ + DROP VIEW IF EXISTS tww_app.vw_additional_wastewater_structure; + + CREATE OR REPLACE VIEW tww_app.vw_additional_wastewater_structure AS + SELECT + ws.identifier as identifier, + + CASE + WHEN wt.obj_id IS NOT NULL THEN 'wwtp_structure' + WHEN sm.obj_id IS NOT NULL THEN 'small_treatment_plant' + WHEN to.obj_id IS NOT NULL THEN 'drainless_toilet' + ELSE 'unknown' + END AS ws_type + + , wt.kind AS wt_kind + , sm.function as sm_function + , ws.fk_owner + , ws.status + + , {ws_cols} + + , main_co_sp.identifier AS co_identifier + , main_co_sp.remark AS co_remark + , main_co_sp.renovation_demand AS co_renovation_demand + + , {main_co_cols} + , ST_Force2D(COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry))::geometry(Point, %(SRID)s) AS situation3d_geometry + + , {wt_columns} + + , {sm_columns} + + , {to_columns} + + , {wn_cols} + , {ne_cols} + + , ws._label + , ws._cover_label + , ws._bottom_label + , ws._input_label + , ws._output_label + , ws._usage_current AS _channel_usage_current + , ws._function_hierarchic AS _channel_function_hierarchic + + FROM tww_od.wastewater_structure ws + LEFT JOIN tww_od.cover main_co ON main_co.obj_id = ws.fk_main_cover + LEFT JOIN tww_od.structure_part main_co_sp ON main_co_sp.obj_id = ws.fk_main_cover + LEFT JOIN tww_od.wwtp_structure wt ON wt.obj_id = ws.obj_id + LEFT JOIN tww_od.small_treatment_plant sm ON sm.obj_id = ws.obj_id + LEFT JOIN tww_od.drainless_toilet to ON to.obj_id = ws.obj_id + LEFT JOIN tww_od.wastewater_networkelement ne ON ne.obj_id = ws.fk_main_wastewater_node + LEFT JOIN tww_od.wastewater_node wn ON wn.obj_id = ws.fk_main_wastewater_node + LEFT JOIN tww_od.channel ch ON ch.obj_id = ws.obj_id + WHERE ch.obj_id IS NULL; + + ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); + ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); + ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); + """.format( + ws_cols=select_columns( + pg_cur=cursor, + table_schema="tww_od", + table_name="wastewater_structure", + table_alias="ws", + remove_pkey=False, + indent=4, + skip_columns=[ + "identifier", + "fk_owner", + "status", + "_label", + "_cover_label", + "_bottom_label", + "_input_label", + "_output_label", + "_usage_current", + "_function_hierarchic", + "fk_main_cover", + "fk_main_wastewater_node", + "detail_geometry3d_geometry", + ], + ), + main_co_cols=select_columns( + pg_cur=cursor, + table_schema="tww_od", + table_name="cover", + table_alias="main_co", + remove_pkey=False, + indent=4, + skip_columns=["situation3d_geometry"], + prefix="co_", + remap_columns={"cover_shape": "co_shape"}, + columns_at_end=["obj_id"], + ), + wt_columns=select_columns( + pg_cur=cursor, + table_schema="tww_od", + table_name="wwtp_structure", + table_alias="wt", + remove_pkey=True, + indent=4, + skip_columns=["kind"], + prefix="wt_", + remap_columns={}, + ), + sm_columns=select_columns( + pg_cur=cursor, + table_schema="tww_od", + table_name="small_treatment_plant", + table_alias="sm", + remove_pkey=True, + indent=4, + skip_columns=["function"], + prefix="sm_", + remap_columns={}, + ), + to_columns=select_columns( + pg_cur=cursor, + table_schema="tww_od", + table_name="drainless_toilet", + table_alias="to", + remove_pkey=True, + indent=4, + skip_columns=[], + prefix="to_", + remap_columns={}, + ), + wn_cols=select_columns( + pg_cur=cursor, + table_schema="tww_od", + table_name="wastewater_node", + table_alias="wn", + remove_pkey=False, + indent=4, + skip_columns=["situation3d_geometry"], + prefix="wn_", + remap_columns={}, + columns_at_end=["obj_id"], + ), + ne_cols=select_columns( + pg_cur=cursor, + table_schema="tww_od", + table_name="wastewater_networkelement", + table_alias="ne", + remove_pkey=True, + indent=4, + skip_columns=[], + prefix="wn_", + remap_columns={}, + ), + ) + + cursor.execute(view_sql, variables) + + trigger_insert_sql = """ + CREATE OR REPLACE FUNCTION tww_app.ft_vw_additional_wastewater_structure_INSERT() + RETURNS trigger AS + $BODY$ + BEGIN + + NEW.identifier = COALESCE(NEW.identifier, NEW.obj_id); + + {insert_ws} + + CASE + WHEN NEW.ws_type = 'wwtp_structure' THEN + -- wwtp_structure + {insert_wt} + + -- small_treatment_plant + WHEN NEW.ws_type = 'small_treatment_plant' THEN + {insert_sm} + + -- drainless_toilet + WHEN NEW.ws_type = 'drainless_toilet' THEN + {insert_to} + + + ELSE + RAISE NOTICE 'Wastewater structure type not known (%)', NEW.ws_type; -- ERROR + END CASE; + + {insert_wn} + + UPDATE tww_od.wastewater_structure + SET fk_main_wastewater_node = NEW.wn_obj_id + WHERE obj_id = NEW.obj_id; + + {insert_vw_cover} + + UPDATE tww_od.wastewater_structure + SET fk_main_cover = NEW.co_obj_id + WHERE obj_id = NEW.obj_id; + + RETURN NEW; + END; $BODY$ LANGUAGE plpgsql VOLATILE; + + DROP TRIGGER IF EXISTS vw_additional_wastewater_structure_INSERT ON tww_app.vw_additional_wastewater_structure; + + CREATE TRIGGER vw_additional_wastewater_structure_INSERT INSTEAD OF INSERT ON tww_app.vw_additional_wastewater_structure + FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_additional_wastewater_structure_INSERT(); + """.format( + insert_ws=insert_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="wastewater_structure", + table_alias="ws", + remove_pkey=False, + indent=2, + skip_columns=[ + "_label", + "_cover_label", + "_bottom_label", + "_input_label", + "_output_label", + "_usage_current", + "_function_hierarchic", + "fk_main_cover", + "fk_main_wastewater_node", + "detail_geometry3d_geometry", + ], + ), + insert_wt=insert_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="wwtp_structure", + table_alias="wt", + prefix="wt_", + remove_pkey=False, + indent=6, + remap_columns={"obj_id": "obj_id"}, + ), + insert_sm=insert_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="small_treatment_plant", + table_alias="sm", + prefix="sm_", + remove_pkey=False, + indent=6, + remap_columns={"obj_id": "obj_id"}, + ), + insert_to=insert_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="drainless_toilet", + table_alias="to", + prefix="to_", + remove_pkey=False, + indent=6, + remap_columns={"obj_id": "obj_id"}, + ), + insert_wn=insert_command( + pg_cur=cursor, + table_schema="tww_app", + table_name="vw_wastewater_node", + table_type="view", + table_alias="wn", + prefix="wn_", + remove_pkey=False, + pkey="obj_id", + indent=6, + insert_values={ + "identifier": "COALESCE(NULLIF(NEW.wn_identifier,''), NEW.identifier)", + "situation3d_geometry": "ST_SetSRID(ST_MakePoint(ST_X(NEW.situation3d_geometry), ST_Y(NEW.situation3d_geometry), 'nan'), {srid} )".format( + srid=srid + ), + "last_modification": "NOW()", + "fk_provider": "COALESCE(NULLIF(NEW.wn_fk_provider,''), NEW.fk_provider)", + "fk_dataowner": "COALESCE(NULLIF(NEW.wn_fk_dataowner,''), NEW.fk_dataowner)", + "fk_wastewater_structure": "NEW.obj_id", + }, + ), + insert_vw_cover=insert_command( + pg_cur=cursor, + table_schema="tww_app", + table_name="vw_cover", + table_type="view", + table_alias="co", + prefix="co_", + remove_pkey=False, + pkey="obj_id", + indent=6, + remap_columns={"cover_shape": "co_shape"}, + insert_values={ + "identifier": "COALESCE(NULLIF(NEW.co_identifier,''), NEW.identifier)", + "situation3d_geometry": "ST_SetSRID(ST_MakePoint(ST_X(NEW.situation3d_geometry), ST_Y(NEW.situation3d_geometry), 'nan'), {srid} )".format( + srid=srid + ), + "last_modification": "NOW()", + "fk_provider": "NEW.fk_provider", + "fk_dataowner": "NEW.fk_dataowner", + "fk_wastewater_structure": "NEW.obj_id", + }, + ), + ) + + cursor.execute(trigger_insert_sql) + + update_trigger_sql = """ + CREATE OR REPLACE FUNCTION tww_app.ft_vw_additional_wastewater_structure_UPDATE() + RETURNS trigger AS + $BODY$ + DECLARE + dx float; + dy float; + BEGIN + {update_co} + {update_sp} + {update_ws} + {update_wn} + + IF OLD.ws_type <> NEW.ws_type THEN + CASE + WHEN OLD.ws_type = 'wwtp_structure' THEN DELETE FROM tww_od.wwtp_structure WHERE obj_id = OLD.obj_id; + WHEN OLD.ws_type = 'small_treatment_plant' THEN DELETE FROM tww_od.small_treatment_plant WHERE obj_id = OLD.obj_id; + WHEN OLD.ws_type = 'drainless_toilet' THEN DELETE FROM tww_od.drainless_toilet WHERE obj_id = OLD.obj_id; + ELSE -- do nothing + END CASE; + + CASE + WHEN NEW.ws_type = 'wwtp_structure' THEN INSERT INTO tww_od.wwtp_structure (obj_id) VALUES(OLD.obj_id); + WHEN NEW.ws_type = 'small_treatment_plant' THEN INSERT INTO tww_od.small_treatment_plant (obj_id) VALUES(OLD.obj_id); + WHEN NEW.ws_type = 'drainless_toilet' THEN INSERT INTO tww_od.drainless_toilet (obj_id) VALUES(OLD.obj_id); + ELSE -- do nothing + END CASE; + END IF; + + CASE + WHEN NEW.ws_type = 'wwtp_structure' THEN + {update_wt} + + WHEN NEW.ws_type = 'small_treatment_plant' THEN + {update_sm} + + WHEN NEW.ws_type = 'drainless_toilet' THEN + {update_to} + + ELSE -- do nothing + END CASE; + + -- Cover geometry has been moved + IF NOT ST_Equals( OLD.situation3d_geometry, NEW.situation3d_geometry) THEN + dx = ST_X(NEW.situation3d_geometry) - ST_X(OLD.situation3d_geometry); + dy = ST_Y(NEW.situation3d_geometry) - ST_Y(OLD.situation3d_geometry); + + -- Move wastewater node as well + -- comment: TRANSLATE((ST_MakePoint(500, 900, 'NaN')), 10, 20, 0) would return NaN NaN NaN - so we have this workaround + UPDATE tww_od.wastewater_node WN + SET situation3d_geometry = ST_SetSRID( ST_MakePoint( + ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )), + ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )), + ST_Z(WN.situation3d_geometry)), %(SRID)s ) + WHERE obj_id IN + ( + SELECT obj_id FROM tww_od.wastewater_networkelement + WHERE fk_wastewater_structure = NEW.obj_id + ); + + -- Move covers + UPDATE tww_od.cover CO + SET situation3d_geometry = ST_SetSRID( ST_MakePoint( + ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )), + ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )), + ST_Z(CO.situation3d_geometry)), %(SRID)s ) + WHERE obj_id IN + ( + SELECT obj_id FROM tww_od.structure_part + WHERE fk_wastewater_structure = NEW.obj_id + ); + + -- Move reach(es) as well + UPDATE tww_od.reach RE + SET progression3d_geometry = + ST_ForceCurve (ST_SetPoint( + ST_CurveToLine (RE.progression3d_geometry ), + 0, -- SetPoint index is 0 based, PointN index is 1 based. + ST_SetSRID( ST_MakePoint( + ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )), + ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )), + ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s ) + ) ) + WHERE fk_reach_point_from IN + ( + SELECT RP.obj_id FROM tww_od.reach_point RP + LEFT JOIN tww_od.wastewater_networkelement NE ON RP.fk_wastewater_networkelement = NE.obj_id + WHERE NE.fk_wastewater_structure = NEW.obj_id + ); + + UPDATE tww_od.reach RE + SET progression3d_geometry = + ST_ForceCurve( ST_SetPoint( + ST_CurveToLine( RE.progression3d_geometry ), + ST_NumPoints(RE.progression3d_geometry) - 1, + ST_SetSRID( ST_MakePoint( + ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )), + ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )), + ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s ) + ) ) + WHERE fk_reach_point_to IN + ( + SELECT RP.obj_id FROM tww_od.reach_point RP + LEFT JOIN tww_od.wastewater_networkelement NE ON RP.fk_wastewater_networkelement = NE.obj_id + WHERE NE.fk_wastewater_structure = NEW.obj_id + ); + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql; + + + + DROP TRIGGER IF EXISTS vw_additional_wastewater_structure_UPDATE ON tww_app.vw_additional_wastewater_structure; + + CREATE TRIGGER vw_additional_wastewater_structure_UPDATE INSTEAD OF UPDATE ON tww_app.vw_additional_wastewater_structure + FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_additional_wastewater_structure_UPDATE(); + """.format( + update_co=update_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="cover", + table_alias="co", + prefix="co_", + indent=6, + skip_columns=["situation3d_geometry"], + remap_columns={"cover_shape": "co_shape"}, + ), + update_sp=update_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="structure_part", + table_alias="sp", + prefix="co_", + indent=6, + skip_columns=["fk_wastewater_structure"], + update_values={ + "last_modification": "NEW.last_modification", + "fk_dataowner": "NEW.fk_dataowner", + "fk_provider": "NEW.fk_provider", + }, + ), + update_ws=update_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="wastewater_structure", + table_alias="ws", + remove_pkey=False, + indent=6, + skip_columns=[ + "detail_geometry3d_geometry", + "last_modification", + "_usage_current", + "_function_hierarchic", + "_label", + "_cover_label", + "_bottom_label", + "_input_label", + "_output_label", + "fk_main_cover", + "fk_main_wastewater_node", + "_depth", + ], + update_values={}, + ), + update_wt=update_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="wwtp_structure", + table_alias="wt", + prefix="wt_", + remove_pkey=True, + indent=6, + skip_columns=[], + remap_columns={"obj_id": "obj_id"}, + ), + update_sm=update_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="small_treatment_plant", + table_alias="sm", + prefix="sm_", + remove_pkey=True, + indent=6, + skip_columns=[], + remap_columns={"obj_id": "obj_id"}, + ), + update_to=update_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="drainless_toilet", + table_alias="to", + prefix="to_", + remove_pkey=True, + indent=6, + skip_columns=[], + remap_columns={"obj_id": "obj_id"}, + ), + update_wn=update_command( + pg_cur=cursor, + table_schema="tww_od", + table_name="wastewater_node", + table_alias="wn", + prefix="wn_", + indent=6, + skip_columns=["situation3d_geometry"], + ), + ) + + cursor.execute(update_trigger_sql, variables) + + trigger_delete_sql = """ + CREATE OR REPLACE FUNCTION tww_app.ft_vw_additional_wastewater_structure_DELETE() + RETURNS trigger AS + $BODY$ + DECLARE + BEGIN + DELETE FROM tww_od.wastewater_structure WHERE obj_id = OLD.obj_id; + RETURN OLD; + END; $BODY$ LANGUAGE plpgsql VOLATILE; + + DROP TRIGGER IF EXISTS vw_additional_wastewater_structure_DELETE ON tww_app.vw_additional_wastewater_structure; + + CREATE TRIGGER vw_additional_wastewater_structure_DELETE INSTEAD OF DELETE ON tww_app.vw_additional_wastewater_structure + FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_additional_wastewater_structure_DELETE(); + """ + cursor.execute(trigger_delete_sql, variables) + + extras = """ + ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); + ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); + ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); + """ + cursor.execute(extras) + + conn.commit() + conn.close() + + +if __name__ == "__main__": + # create the top-level parser + parser = argparse.ArgumentParser() + parser.add_argument("-s", "--srid", help="EPSG code for SRID") + parser.add_argument("-p", "--pg_service", help="the PostgreSQL service name") + args = parser.parse_args() + srid = args.srid or os.getenv("SRID") + pg_service = args.pg_service or os.getenv("PGSERVICE") + vw_additional_wastewater_structure( + srid=srid, pg_service=pg_service + ) From f58658369ef6370e68fe31d7122b2b366e30e3b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:17:10 +0000 Subject: [PATCH 02/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/create_views.py | 3 +-- datamodel/app/view/vw_additional_wastewater_structure.py | 9 +++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index dfd75b373..f8455e455 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -183,8 +183,7 @@ def create_views( srid, pg_service=pg_service, extra_definition=tww_wastewater_structure_extra ) vw_tww_reach(pg_service=pg_service, extra_definition=tww_reach_extra) - vw_additional_wastewater_structure( - srid, pg_service=pg_service) + vw_additional_wastewater_structure(srid, pg_service=pg_service) run_sql("app/view/vw_file.sql", pg_service, variables) MultipleInheritance( diff --git a/datamodel/app/view/vw_additional_wastewater_structure.py b/datamodel/app/view/vw_additional_wastewater_structure.py index 64605c98a..1745b8306 100644 --- a/datamodel/app/view/vw_additional_wastewater_structure.py +++ b/datamodel/app/view/vw_additional_wastewater_structure.py @@ -6,8 +6,7 @@ import os import psycopg2 -from pirogue.utils import insert_command, select_columns, table_parts, update_command -from yaml import safe_load +from pirogue.utils import insert_command, select_columns, update_command def vw_additional_wastewater_structure(srid: int, pg_service: str = None): @@ -35,7 +34,7 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): CASE WHEN wt.obj_id IS NOT NULL THEN 'wwtp_structure' WHEN sm.obj_id IS NOT NULL THEN 'small_treatment_plant' - WHEN to.obj_id IS NOT NULL THEN 'drainless_toilet' + WHEN to.obj_id IS NOT NULL THEN 'drainless_toilet' ELSE 'unknown' END AS ws_type @@ -574,6 +573,4 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): args = parser.parse_args() srid = args.srid or os.getenv("SRID") pg_service = args.pg_service or os.getenv("PGSERVICE") - vw_additional_wastewater_structure( - srid=srid, pg_service=pg_service - ) + vw_additional_wastewater_structure(srid=srid, pg_service=pg_service) From 70a47134e3b6749f089f35907c73f479e8770061 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:18:35 +0100 Subject: [PATCH 03/48] might help to import the code you just added --- datamodel/app/view/create_views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index dfd75b373..0f71885ac 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -6,6 +6,7 @@ from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance from vw_tww_reach import vw_tww_reach from vw_tww_wastewater_structure import vw_tww_wastewater_structure +from vw_additional_wastewater_structure import vw_additional_wastewater_structure from yaml import safe_load # sys.path.append(os.path.join(os.path.dirname(__file__))) From d739484f597b5e139d0b0219ab5d23b5c0b29e8a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:19:34 +0000 Subject: [PATCH 04/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/create_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index d3188cdc9..87375ed64 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -4,9 +4,9 @@ import psycopg2 from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance +from vw_additional_wastewater_structure import vw_additional_wastewater_structure from vw_tww_reach import vw_tww_reach from vw_tww_wastewater_structure import vw_tww_wastewater_structure -from vw_additional_wastewater_structure import vw_additional_wastewater_structure from yaml import safe_load # sys.path.append(os.path.join(os.path.dirname(__file__))) From b4a87cf7526d008957ca0962db0b0a8fcd2b36ae Mon Sep 17 00:00:00 2001 From: Denis Rouzaud Date: Wed, 6 Dec 2023 23:45:57 +0100 Subject: [PATCH 05/48] Update datamodel-test.yml --- .github/workflows/datamodel-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/datamodel-test.yml b/.github/workflows/datamodel-test.yml index d8a61e3e3..fa5b0c5eb 100644 --- a/.github/workflows/datamodel-test.yml +++ b/.github/workflows/datamodel-test.yml @@ -39,6 +39,10 @@ jobs: - name: run tests run: docker exec teksi-wastewater pytest + - name: logs + if: failure() + run: docker logs teksi-wastewater + static-tests: name: Run static tests on datamodel runs-on: ubuntu-latest From f31267e226b843d12e1811eadd2c63f165791b2c Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:02:43 +0100 Subject: [PATCH 06/48] Alter drainless_toilet alias and improve import view This push - Alters the view name from vw_additional_wastewater_structure to vw_tww_additional_ws - renames the drainless_toilet alias - alters the import view to not depend on vw_tww_wastewater_structure during creation (allows re-calling create_views.py without dependency on existing views) --- datamodel/app/view/create_views.py | 4 +- ...r_structure.py => vw_tww_additional_ws.py} | 68 +++++++++---------- datamodel/app/view/vw_tww_import.sql | 56 +++++++++------ 3 files changed, 72 insertions(+), 56 deletions(-) rename datamodel/app/view/{vw_additional_wastewater_structure.py => vw_tww_additional_ws.py} (86%) diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index 0f7a23d45..ab7c584a0 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -4,7 +4,7 @@ import psycopg2 from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance -from vw_additional_wastewater_structure import vw_additional_wastewater_structure +from vw_tww_additional_ws import vw_tww_additional_ws from vw_tww_reach import vw_tww_reach from vw_tww_wastewater_structure import vw_tww_wastewater_structure from yaml import safe_load @@ -195,7 +195,7 @@ def create_views( srid, pg_service=pg_service, extra_definition=tww_wastewater_structure_extra ) vw_tww_reach(pg_service=pg_service, extra_definition=tww_reach_extra) - vw_additional_wastewater_structure(srid, pg_service=pg_service) + vw_tww_additional_ws(srid, pg_service=pg_service) run_sql("app/view/vw_file.sql", pg_service, variables) MultipleInheritance( diff --git a/datamodel/app/view/vw_additional_wastewater_structure.py b/datamodel/app/view/vw_tww_additional_ws.py similarity index 86% rename from datamodel/app/view/vw_additional_wastewater_structure.py rename to datamodel/app/view/vw_tww_additional_ws.py index 1745b8306..9f1961609 100644 --- a/datamodel/app/view/vw_additional_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# -- View: vw_additional_wastewater_structure +# -- View: vw_tww_additional_ws import argparse import os @@ -9,7 +9,7 @@ from pirogue.utils import insert_command, select_columns, update_command -def vw_additional_wastewater_structure(srid: int, pg_service: str = None): +def vw_tww_additional_ws(srid: int, pg_service: str = None): """ Creates additional_wastewater_structure view :param srid: EPSG code for geometries @@ -25,9 +25,9 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): cursor = conn.cursor() view_sql = """ - DROP VIEW IF EXISTS tww_app.vw_additional_wastewater_structure; + DROP VIEW IF EXISTS tww_app.vw_tww_additional_ws; - CREATE OR REPLACE VIEW tww_app.vw_additional_wastewater_structure AS + CREATE OR REPLACE VIEW tww_app.vw_tww_additional_ws AS SELECT ws.identifier as identifier, @@ -56,7 +56,7 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): , {sm_columns} - , {to_columns} + , {dt_columns} , {wn_cols} , {ne_cols} @@ -80,9 +80,9 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): LEFT JOIN tww_od.channel ch ON ch.obj_id = ws.obj_id WHERE ch.obj_id IS NULL; - ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); - ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); - ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); + ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); + ALTER VIEW tww_app.vw_tww_additional_ws ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); + ALTER VIEW tww_app.vw_tww_additional_ws ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); """.format( ws_cols=select_columns( pg_cur=cursor, @@ -141,11 +141,11 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): prefix="sm_", remap_columns={}, ), - to_columns=select_columns( + dt_columns=select_columns( pg_cur=cursor, table_schema="tww_od", table_name="drainless_toilet", - table_alias="to", + table_alias="dt", remove_pkey=True, indent=4, skip_columns=[], @@ -180,7 +180,7 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): cursor.execute(view_sql, variables) trigger_insert_sql = """ - CREATE OR REPLACE FUNCTION tww_app.ft_vw_additional_wastewater_structure_INSERT() + CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_additional_ws_INSERT() RETURNS trigger AS $BODY$ BEGIN @@ -200,7 +200,7 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): -- drainless_toilet WHEN NEW.ws_type = 'drainless_toilet' THEN - {insert_to} + {insert_dt} ELSE @@ -222,10 +222,10 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): RETURN NEW; END; $BODY$ LANGUAGE plpgsql VOLATILE; - DROP TRIGGER IF EXISTS vw_additional_wastewater_structure_INSERT ON tww_app.vw_additional_wastewater_structure; + DROP TRIGGER IF EXISTS vw_tww_additional_ws_INSERT ON tww_app.vw_tww_additional_ws; - CREATE TRIGGER vw_additional_wastewater_structure_INSERT INSTEAD OF INSERT ON tww_app.vw_additional_wastewater_structure - FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_additional_wastewater_structure_INSERT(); + CREATE TRIGGER vw_tww_additional_ws_INSERT INSTEAD OF INSERT ON tww_app.vw_tww_additional_ws + FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_additional_ws_INSERT(); """.format( insert_ws=insert_command( pg_cur=cursor, @@ -267,12 +267,12 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): indent=6, remap_columns={"obj_id": "obj_id"}, ), - insert_to=insert_command( + insert_dt=insert_command( pg_cur=cursor, table_schema="tww_od", table_name="drainless_toilet", - table_alias="to", - prefix="to_", + table_alias="dt", + prefix="dt_", remove_pkey=False, indent=6, remap_columns={"obj_id": "obj_id"}, @@ -325,7 +325,7 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): cursor.execute(trigger_insert_sql) update_trigger_sql = """ - CREATE OR REPLACE FUNCTION tww_app.ft_vw_additional_wastewater_structure_UPDATE() + CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_additional_ws_UPDATE() RETURNS trigger AS $BODY$ DECLARE @@ -361,7 +361,7 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): {update_sm} WHEN NEW.ws_type = 'drainless_toilet' THEN - {update_to} + {update_dt} ELSE -- do nothing END CASE; @@ -439,10 +439,10 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): - DROP TRIGGER IF EXISTS vw_additional_wastewater_structure_UPDATE ON tww_app.vw_additional_wastewater_structure; + DROP TRIGGER IF EXISTS vw_tww_additional_ws_UPDATE ON tww_app.vw_tww_additional_ws; - CREATE TRIGGER vw_additional_wastewater_structure_UPDATE INSTEAD OF UPDATE ON tww_app.vw_additional_wastewater_structure - FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_additional_wastewater_structure_UPDATE(); + CREATE TRIGGER vw_tww_additional_ws_UPDATE INSTEAD OF UPDATE ON tww_app.vw_tww_additional_ws + FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_additional_ws_UPDATE(); """.format( update_co=update_command( pg_cur=cursor, @@ -513,12 +513,12 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): skip_columns=[], remap_columns={"obj_id": "obj_id"}, ), - update_to=update_command( + update_dt=update_command( pg_cur=cursor, table_schema="tww_od", table_name="drainless_toilet", - table_alias="to", - prefix="to_", + table_alias="dt", + prefix="dt_", remove_pkey=True, indent=6, skip_columns=[], @@ -538,7 +538,7 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): cursor.execute(update_trigger_sql, variables) trigger_delete_sql = """ - CREATE OR REPLACE FUNCTION tww_app.ft_vw_additional_wastewater_structure_DELETE() + CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_additional_ws_DELETE() RETURNS trigger AS $BODY$ DECLARE @@ -547,17 +547,17 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): RETURN OLD; END; $BODY$ LANGUAGE plpgsql VOLATILE; - DROP TRIGGER IF EXISTS vw_additional_wastewater_structure_DELETE ON tww_app.vw_additional_wastewater_structure; + DROP TRIGGER IF EXISTS vw_tww_additional_ws_DELETE ON tww_app.vw_tww_additional_ws; - CREATE TRIGGER vw_additional_wastewater_structure_DELETE INSTEAD OF DELETE ON tww_app.vw_additional_wastewater_structure - FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_additional_wastewater_structure_DELETE(); + CREATE TRIGGER vw_tww_additional_ws_DELETE INSTEAD OF DELETE ON tww_app.vw_tww_additional_ws + FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_additional_ws_DELETE(); """ cursor.execute(trigger_delete_sql, variables) extras = """ - ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); - ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); - ALTER VIEW tww_app.vw_additional_wastewater_structure ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); + ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); + ALTER VIEW tww_app.vw_tww_additional_ws ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); + ALTER VIEW tww_app.vw_tww_additional_ws ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); """ cursor.execute(extras) @@ -573,4 +573,4 @@ def vw_additional_wastewater_structure(srid: int, pg_service: str = None): args = parser.parse_args() srid = args.srid or os.getenv("SRID") pg_service = args.pg_service or os.getenv("PGSERVICE") - vw_additional_wastewater_structure(srid=srid, pg_service=pg_service) + vw_tww_additional_ws(srid=srid, pg_service=pg_service) diff --git a/datamodel/app/view/vw_tww_import.sql b/datamodel/app/view/vw_tww_import.sql index 63a3ef11d..f2a60a6f8 100644 --- a/datamodel/app/view/vw_tww_import.sql +++ b/datamodel/app/view/vw_tww_import.sql @@ -3,22 +3,28 @@ CREATE OR REPLACE VIEW tww_app.import_vw_manhole AS SELECT DISTINCT ON (ws.obj_id) ws.obj_id, ws.identifier, - ST_Force3D(ws.situation3d_geometry)::geometry(POINTZ, %(SRID)s) AS situation_geometry, - ws.co_shape, - ws.co_diameter, - ws.co_material, - ws.co_positional_accuracy, - ws.co_level, + COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry)::geometry(POINTZ, %(SRID)s) AS situation_geometry, + main_co.cover_shape as co_shape, + main_co.diameter as co_diameter, + main_co.material as co_material, + main_co.positional_accuracy as co_positional_accuracy, + main_co.level as co_level, ws._depth::numeric(6, 3) AS _depth, - ws._channel_usage_current, - ws.ma_material, - ws.ma_dimension1, - ws.ma_dimension2, - ws.ws_type, - ws.ma_function, - ws.ss_function, - ws.remark, - ws.wn_bottom_level, + ws._usage_current AS _channel_usage_current, + ma.material as ma_material, + ma.dimension1 as ma_dimension1, + ma.dimension2 as ma_dimension2, + CASE + WHEN ma.obj_id IS NOT NULL THEN 'manhole' + WHEN ss.obj_id IS NOT NULL THEN 'special_structure' + WHEN dp.obj_id IS NOT NULL THEN 'discharge_point' + WHEN ii.obj_id IS NOT NULL THEN 'infiltration_installation' + ELSE 'unknown' + END AS ws_type, + ma.function as ma_function, + ss.function as ss_function, + ne.remark, + wn.bottom_level as wn_bottom_level, NULL::text AS photo1, NULL::text AS photo2, NULL::smallint AS inlet_3_material, @@ -43,13 +49,23 @@ CREATE OR REPLACE VIEW tww_app.import_vw_manhole AS NULL::integer AS outlet_2_clear_height, NULL::numeric(7, 3) AS outlet_2_depth_m, FALSE::boolean AS verified, - (CASE WHEN EXISTS ( SELECT TRUE FROM tww_od.import_manhole_quarantine q WHERE q.obj_id = ws.obj_id ) - THEN TRUE - ELSE FALSE - END) AS in_quarantine, + (q.obj_id IS NOT NULL) AS in_quarantine, FALSE::boolean AS deleted - FROM tww_app.vw_tww_wastewater_structure ws; + FROM tww_od.wastewater_structure ws + LEFT JOIN tww_od.cover main_co ON main_co.obj_id = ws.fk_main_cover + LEFT JOIN tww_od.manhole ma ON ma.obj_id = ws.obj_id + LEFT JOIN tww_od.special_structure ss ON ss.obj_id = ws.obj_id + LEFT JOIN tww_od.discharge_point dp ON dp.obj_id = ws.obj_id + LEFT JOIN tww_od.infiltration_installation ii ON ii.obj_id = ws.obj_id + LEFT JOIN tww_od.wastewater_networkelement ne ON ne.obj_id = ws.fk_main_wastewater_node + LEFT JOIN tww_od.wastewater_node wn ON wn.obj_id = ws.fk_main_wastewater_node + LEFT JOIN (SELECT DISTINCT obj_id + , max(quarantine_serial) as maximum + FROM tww_od.import_manhole_quarantine + GROUP BY obj_id + ) q ON q.obj_id = ws.obj_id + ; -- create trigger function and trigger for mobile view From 47cb1e4830937b860074ca7779b5a94c87fce2fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 08:02:52 +0000 Subject: [PATCH 07/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/vw_tww_import.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamodel/app/view/vw_tww_import.sql b/datamodel/app/view/vw_tww_import.sql index f2a60a6f8..a0255eeaf 100644 --- a/datamodel/app/view/vw_tww_import.sql +++ b/datamodel/app/view/vw_tww_import.sql @@ -52,7 +52,7 @@ CREATE OR REPLACE VIEW tww_app.import_vw_manhole AS (q.obj_id IS NOT NULL) AS in_quarantine, FALSE::boolean AS deleted - FROM tww_od.wastewater_structure ws + FROM tww_od.wastewater_structure ws LEFT JOIN tww_od.cover main_co ON main_co.obj_id = ws.fk_main_cover LEFT JOIN tww_od.manhole ma ON ma.obj_id = ws.obj_id LEFT JOIN tww_od.special_structure ss ON ss.obj_id = ws.obj_id From 497dba7e53a7f1a4399ee9003d0c943a0869e13c Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:06:40 +0100 Subject: [PATCH 08/48] alter the aliases I missed --- datamodel/app/view/vw_tww_additional_ws.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 9f1961609..ac12361b6 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -34,11 +34,11 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): CASE WHEN wt.obj_id IS NOT NULL THEN 'wwtp_structure' WHEN sm.obj_id IS NOT NULL THEN 'small_treatment_plant' - WHEN to.obj_id IS NOT NULL THEN 'drainless_toilet' + WHEN dt.obj_id IS NOT NULL THEN 'drainless_toilet' ELSE 'unknown' END AS ws_type - , wt.kind AS wt_kind + , sm.function as sm_function , ws.fk_owner , ws.status @@ -74,7 +74,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): LEFT JOIN tww_od.structure_part main_co_sp ON main_co_sp.obj_id = ws.fk_main_cover LEFT JOIN tww_od.wwtp_structure wt ON wt.obj_id = ws.obj_id LEFT JOIN tww_od.small_treatment_plant sm ON sm.obj_id = ws.obj_id - LEFT JOIN tww_od.drainless_toilet to ON to.obj_id = ws.obj_id + LEFT JOIN tww_od.drainless_toilet dt ON dt.obj_id = ws.obj_id LEFT JOIN tww_od.wastewater_networkelement ne ON ne.obj_id = ws.fk_main_wastewater_node LEFT JOIN tww_od.wastewater_node wn ON wn.obj_id = ws.fk_main_wastewater_node LEFT JOIN tww_od.channel ch ON ch.obj_id = ws.obj_id @@ -149,7 +149,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): remove_pkey=True, indent=4, skip_columns=[], - prefix="to_", + prefix="dt_", remap_columns={}, ), wn_cols=select_columns( From 7538e2025a05fb72c6eea52cf398ecaaf8743ed6 Mon Sep 17 00:00:00 2001 From: Arnaud Poncet-Montanges Date: Thu, 7 Dec 2023 10:02:49 +0100 Subject: [PATCH 09/48] Update .github/workflows/datamodel-test.yml --- .github/workflows/datamodel-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datamodel-test.yml b/.github/workflows/datamodel-test.yml index fa5b0c5eb..03ef13ff9 100644 --- a/.github/workflows/datamodel-test.yml +++ b/.github/workflows/datamodel-test.yml @@ -39,7 +39,7 @@ jobs: - name: run tests run: docker exec teksi-wastewater pytest - - name: logs + - name: failure logs if: failure() run: docker logs teksi-wastewater From 433cf4ce514f5834164d227677f372540b3e2e3d Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:06:14 +0100 Subject: [PATCH 10/48] alter prefixes back --- datamodel/app/view/vw_tww_additional_ws.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index ac12361b6..9c080117e 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -149,7 +149,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): remove_pkey=True, indent=4, skip_columns=[], - prefix="dt_", + prefix="to_", remap_columns={}, ), wn_cols=select_columns( @@ -272,7 +272,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): table_schema="tww_od", table_name="drainless_toilet", table_alias="dt", - prefix="dt_", + prefix="to_", remove_pkey=False, indent=6, remap_columns={"obj_id": "obj_id"}, @@ -518,7 +518,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): table_schema="tww_od", table_name="drainless_toilet", table_alias="dt", - prefix="dt_", + prefix="to_", remove_pkey=True, indent=6, skip_columns=[], From f20278ac51f69e1a1baa80b68a6f3dfb0186a838 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:21:04 +0100 Subject: [PATCH 11/48] alter where filters --- datamodel/app/view/vw_tww_additional_ws.py | 6 +++++- datamodel/app/view/vw_tww_wastewater_structure.py | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 9c080117e..550568440 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -78,7 +78,11 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): LEFT JOIN tww_od.wastewater_networkelement ne ON ne.obj_id = ws.fk_main_wastewater_node LEFT JOIN tww_od.wastewater_node wn ON wn.obj_id = ws.fk_main_wastewater_node LEFT JOIN tww_od.channel ch ON ch.obj_id = ws.obj_id - WHERE ch.obj_id IS NULL; + LEFT JOIN tww_od.manhole ma ON ma.obj_id = ws.obj_id + LEFT JOIN tww_od.special_structure ss ON ss.obj_id = ws.obj_id + LEFT JOIN tww_od.discharge_point dp ON dp.obj_id = ws.obj_id + LEFT JOIN tww_od.infiltration_installation ii ON ii.obj_id = ws.obj_id + WHERE NULL = ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]); ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); ALTER VIEW tww_app.vw_tww_additional_ws ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index bfdcaea42..a40a02d0c 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -86,9 +86,12 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit LEFT JOIN tww_od.infiltration_installation ii ON ii.obj_id = ws.obj_id LEFT JOIN tww_od.wastewater_networkelement ne ON ne.obj_id = ws.fk_main_wastewater_node LEFT JOIN tww_od.wastewater_node wn ON wn.obj_id = ws.fk_main_wastewater_node - LEFT JOIN tww_od.channel ch ON ch.obj_id = ws.obj_id {extra_joins} - WHERE ch.obj_id IS NULL; + LEFT JOIN tww_od.channel ch ON ch.obj_id = ws.obj_id + LEFT JOIN tww_od.wwtp_structure wt ON wt.obj_id = ws.obj_id + LEFT JOIN tww_od.small_treatment_plant sm ON sm.obj_id = ws.obj_id + LEFT JOIN tww_od.drainless_toilet dt ON dt.obj_id = ws.obj_id + WHERE NULL = ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]); ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); From 25e6aa56d3e626a9cd6b99f0ef3788cbb4252b48 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Wed, 13 Dec 2023 08:46:10 +0100 Subject: [PATCH 12/48] alter type check Where clauses do not check against NULL --- datamodel/app/view/vw_tww_additional_ws.py | 3 ++- datamodel/app/view/vw_tww_wastewater_structure.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 550568440..bf0bc3883 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -82,7 +82,8 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): LEFT JOIN tww_od.special_structure ss ON ss.obj_id = ws.obj_id LEFT JOIN tww_od.discharge_point dp ON dp.obj_id = ws.obj_id LEFT JOIN tww_od.infiltration_installation ii ON ii.obj_id = ws.obj_id - WHERE NULL = ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]); + WHERE '-1'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL + AND '-2'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); ALTER VIEW tww_app.vw_tww_additional_ws ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index a40a02d0c..41dbf5871 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -91,7 +91,8 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit LEFT JOIN tww_od.wwtp_structure wt ON wt.obj_id = ws.obj_id LEFT JOIN tww_od.small_treatment_plant sm ON sm.obj_id = ws.obj_id LEFT JOIN tww_od.drainless_toilet dt ON dt.obj_id = ws.obj_id - WHERE NULL = ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]); + WHERE '-1'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL + AND '-2'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); From c7e27c33fd4f3c54cfc0ab535c25fc38496b3ab5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Dec 2023 07:48:01 +0000 Subject: [PATCH 13/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/vw_tww_additional_ws.py | 2 +- datamodel/app/view/vw_tww_wastewater_structure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index bf0bc3883..363cee802 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -82,7 +82,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): LEFT JOIN tww_od.special_structure ss ON ss.obj_id = ws.obj_id LEFT JOIN tww_od.discharge_point dp ON dp.obj_id = ws.obj_id LEFT JOIN tww_od.infiltration_installation ii ON ii.obj_id = ws.obj_id - WHERE '-1'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL + WHERE '-1'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL AND '-2'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 41dbf5871..17fbda988 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -91,7 +91,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit LEFT JOIN tww_od.wwtp_structure wt ON wt.obj_id = ws.obj_id LEFT JOIN tww_od.small_treatment_plant sm ON sm.obj_id = ws.obj_id LEFT JOIN tww_od.drainless_toilet dt ON dt.obj_id = ws.obj_id - WHERE '-1'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL + WHERE '-1'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL AND '-2'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); From e4d25132b391938eba2cf1dac5901558cee085ba Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Wed, 13 Dec 2023 08:56:59 +0100 Subject: [PATCH 14/48] replace tabs with spaces --- datamodel/app/view/vw_tww_additional_ws.py | 2 +- datamodel/app/view/vw_tww_wastewater_structure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index bf0bc3883..40d401245 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -83,7 +83,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): LEFT JOIN tww_od.discharge_point dp ON dp.obj_id = ws.obj_id LEFT JOIN tww_od.infiltration_installation ii ON ii.obj_id = ws.obj_id WHERE '-1'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL - AND '-2'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL; + AND '-2'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); ALTER VIEW tww_app.vw_tww_additional_ws ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 41dbf5871..878b961a2 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -92,7 +92,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit LEFT JOIN tww_od.small_treatment_plant sm ON sm.obj_id = ws.obj_id LEFT JOIN tww_od.drainless_toilet dt ON dt.obj_id = ws.obj_id WHERE '-1'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL - AND '-2'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL; + AND '-2'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); From 365fa941a3f5edf8baaf15bf342f2db45804b31e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Dec 2023 07:58:29 +0000 Subject: [PATCH 15/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/vw_tww_additional_ws.py | 2 +- datamodel/app/view/vw_tww_wastewater_structure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 40d401245..c697335f7 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -82,7 +82,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): LEFT JOIN tww_od.special_structure ss ON ss.obj_id = ws.obj_id LEFT JOIN tww_od.discharge_point dp ON dp.obj_id = ws.obj_id LEFT JOIN tww_od.infiltration_installation ii ON ii.obj_id = ws.obj_id - WHERE '-1'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL + WHERE '-1'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL AND '-2'= ALL(ARRAY[ch.obj_id,ma.obj_id,ss.obj_id,dp.obj_id,ii.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 878b961a2..1fd0292ef 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -91,7 +91,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit LEFT JOIN tww_od.wwtp_structure wt ON wt.obj_id = ws.obj_id LEFT JOIN tww_od.small_treatment_plant sm ON sm.obj_id = ws.obj_id LEFT JOIN tww_od.drainless_toilet dt ON dt.obj_id = ws.obj_id - WHERE '-1'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL + WHERE '-1'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL AND '-2'=ALL(ARRAY[ch.obj_id,dt.obj_id,sm.obj_id,wt.obj_id]) IS NULL; ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); From 6ac8646c3ef0ca712d7b4d246280bbdb7d9a4e57 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:18:46 +0100 Subject: [PATCH 16/48] add new view to test scripts I didn't find out where they are called --- datamodel/test/test_geometry.py | 178 ++++++++++++++++++++++++++++++++ datamodel/test/test_views.py | 38 +++++++ 2 files changed, 216 insertions(+) diff --git a/datamodel/test/test_geometry.py b/datamodel/test/test_geometry.py index 581474e88..1bc072ffe 100644 --- a/datamodel/test/test_geometry.py +++ b/datamodel/test/test_geometry.py @@ -632,6 +632,184 @@ def test_cover_geometry_sync_on_update(self): == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" ) assert new_row["level"] is None + + def test_vw_tww_additional_ws_geometry_insert(self): + # 1. insert geometry and no co_level and no wn_bottom_level + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1001', '1337_1001'); + row = { + "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", + "wn_obj_id": "1337_1001", + "co_obj_id": "1337_1001", + } + expected_row = copy.deepcopy(row) + # wastewaterstructure has the geometry but NaN as Z because of no co_level (geometry of cover): ST_SetSRID(ST_Collect(ST_MakePoint(2600000, 1200000, 'NaN')), 2056) + expected_row["situation3d_geometry"] = "0101000020080800000000000020D6434100000000804F3241" + # co_level is NULL + expected_row["co_level"] = None + # wn_bottom_level NULL + expected_row["wn_bottom_level"] = None + self.insert_check("vw_tww_additional_ws", row, expected_row) + # cover geometry has the geometry but NaN as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 'NaN'), 2056) + row = self.select("cover", "1337_1001", schema="tww_od") + assert ( + row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" + ) + # wastewater_node has the geometry but not 3d: ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056) + row = self.select("wastewater_node", "1337_1001", schema="tww_od") + assert row["situation3d_geometry"] == self.execute( + "ST_SetSRID( ST_MakePoint(2600000, 1200000, 'NaN'), 2056)" + ) + + # 2. insert geometry with no co_level and WITH wn_bottom_level + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id, wn_bottom_level) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1002', '1337_1002', 200.000); + row = { + "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", + "wn_obj_id": "1337_1002", + "co_obj_id": "1337_1002", + "wn_bottom_level": "200.000", + } + expected_row = copy.deepcopy(row) + # wastewaterstructure has the 2D geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056) + expected_row["situation3d_geometry"] = "0101000020080800000000000020D6434100000000804F3241" + # co_level is NULL + expected_row["co_level"] = None + # wn_bottom_level is new wn_bottom_level + expected_row["wn_bottom_level"] = "200.000" + self.insert_check("vw_tww_additional_ws", row, expected_row) + # cover geometry has the geometry but NaN as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 'NaN'), 2056) + row = self.select("cover", "1337_1002", schema="tww_od") + assert ( + row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" + ) + # wastewater_node has the geometry and wn_buttom_level as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) + row = self.select("wastewater_node", "1337_1002", schema="tww_od") + assert ( + row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000006940" + ) + + # 3. insert geometry with Z and WITH co_level and WITH wn_bottom_level + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id, wn_bottom_level, co_level) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1003', '1337_1003', 200.000, 500.000); + row = { + "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", + "wn_obj_id": "1337_1003", + "co_obj_id": "1337_1003", + "wn_bottom_level": "200.000", + "co_level": "500.000", + } + expected_row = copy.deepcopy(row) + # wastewaterstructure has 2D geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056) + expected_row["situation3d_geometry"] = "0101000020080800000000000020D6434100000000804F3241" + # co_level is new co_level + expected_row["co_level"] = "500.000" + # wn_bottom_level is new wn_bottom_level + expected_row["wn_bottom_level"] = "200.000" + self.insert_check("vw_tww_additional_ws", row, expected_row) + # cover geometry has the geometry and co_level as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 500), 2056) + row = self.select("cover", "1337_1003", schema="tww_od") + assert ( + row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000407F40" + ) + # wastewater_node has the geometry and wn_buttom_level as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) + row = self.select("wastewater_node", "1337_1003", schema="tww_od") + assert ( + row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000006940" + ) + + def test_vw_tww_additional_ws_geometry_update(self): + # first insert + # insert geometry with no co_level and no wn_bottom_level + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1010', '1337_1010'); + row = { + "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", + "ws_type": "manhole", + "wn_obj_id": "1337_1010", + "co_obj_id": "1337_1010", + } + obj_id = self.insert("vw_tww_additional_ws", row) + + # 1. update no change on geometry with Z but WITH wn_bottom_level + # UPDATE INTO tww_app.vw_wastewater_node SET wn_bottom_level=200.000 WHERE obj_id = obj_id + row = {"wn_bottom_level": "200.000"} + self.update("vw_tww_additional_ws", row, obj_id) + new_row = self.select("vw_tww_additional_ws", obj_id) + # no change on geometry of wastewaterstructure (because it's geometry of cover that does not change) + self.assertEqual( + new_row["situation3d_geometry"], "0101000020080800000000000020D6434100000000804F3241" + ) + # no change on co_level + self.assertIsNone(new_row["co_level"]) + # wn_bottom_level is new wn_bottom_level + self.assertEqual(new_row["wn_bottom_level"], 200.000) + # no change on cover geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000, 'NaN'), 2056) + new_row = self.select("cover", "1337_1010", schema="tww_od") + assert ( + new_row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" + ) + # wastewater_node geometry has Z from new wn_bottom_level: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) + new_row = self.select("wastewater_node", "1337_1010", schema="tww_od") + assert ( + new_row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000006940" + ) + + # 2. update change co_level + # UPDATE INTO tww_app.vw_wastewater_node SET level=500.000 WHERE obj_id = obj_id + row = {"co_level": "500.000"} + self.update("vw_tww_additional_ws", row, obj_id) + new_row = self.select("vw_tww_additional_ws", obj_id) + # geometry of wastewaterstructure has 2D geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000, 500), 2056) + assert ( + new_row["situation3d_geometry"] == "0101000020080800000000000020D6434100000000804F3241" + ) + # co_level is new co_level + assert new_row["co_level"] == 500.000 + # no change on wn_bottom_level + assert new_row["wn_bottom_level"] == 200.000 + # cover geometry has Z from new co_level: ST_SetSRID(ST_MakePoint(2600000, 1200000, 500), 2056) + new_row = self.select("cover", "1337_1010", schema="tww_od") + assert ( + new_row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000407F40" + ) + # no change on wastewater_node geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) + new_row = self.select("wastewater_node", "1337_1010", schema="tww_od") + assert ( + new_row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000006940" + ) + + # 3. update change WITH co_level and WITH wn_bottom_level + # UPDATE INTO tww_app.vw_wastewater_node SET co_level=600.000, wn_bottom_level=300.000 WHERE obj_id = obj_id + row = {"co_level": "600.000", "wn_bottom_level": "300.000"} + self.update("vw_tww_additional_ws", row, obj_id) + new_row = self.select("vw_tww_additional_ws", obj_id) + # geometry of wastewaterstructure unchanged: ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056) + assert ( + new_row["situation3d_geometry"] == "0101000020080800000000000020D6434100000000804F3241" + ) + # co_level is new co_level + assert new_row["co_level"] == 600.000 + # wn_bottom_level is new wn_bottom_level + assert new_row["wn_bottom_level"] == 300.000 + # cover geometry has Z from new co_level: ST_SetSRID(ST_MakePoint(2600000, 1200000, 600), 2056) + new_row = self.select("cover", "1337_1010", schema="tww_od") + assert ( + new_row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000C08240" + ) + # no change on wastewater_node geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000, 300), 2056) + new_row = self.select("wastewater_node", "1337_1010", schema="tww_od") + assert ( + new_row["situation3d_geometry"] + == "01010000A0080800000000000020D6434100000000804F32410000000000C07240" + ) + if __name__ == "__main__": diff --git a/datamodel/test/test_views.py b/datamodel/test/test_views.py index d2acaaba5..adedd5ce5 100644 --- a/datamodel/test/test_views.py +++ b/datamodel/test/test_views.py @@ -103,6 +103,44 @@ def test_vw_tww_wastewater_structure(self): row = cur.fetchone() assert row["backflow_level_current"] == decimal.Decimal("100.000") + + def test_vw_tww_additional_ws(self): + row = { + "identifier": "20", + "ws_type": "drainless_toilet", + "situation3d_geometry": self.execute( + "ST_SetSRID(ST_GeomFromText('POINT(2600000 1200000)'), 2056)" + ), + # 'co_material': 5355, + "wn_backflow_level_current": decimal.Decimal("100.000"), + } + + expected_row = copy.deepcopy(row) + expected_row["situation3d_geometry"] = self.execute( + "ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056)" + ) + + obj_id = self.insert_check("vw_tww_additional_ws", row, expected_row) + + row = { + "identifier": "10", + "ws_type": "wwtp_structure", + "co_material": 233, + "wt_kind": 331, + } + + self.update_check("vw_tww_wastewater_structure", row, obj_id) + + cur = self.cursor() + + cur.execute( + "SELECT * FROM tww_od.wastewater_networkelement NE LEFT JOIN tww_od.wastewater_node NO ON NO.obj_id = NE.obj_id WHERE fk_wastewater_structure='{obj_id}' ".format( + obj_id=obj_id + ) + ) + row = cur.fetchone() + + assert row["backflow_level_current"] == decimal.Decimal("100.000") if __name__ == "__main__": From d96e9a27698564b7bd1c425d2067bf6ad60260e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Dec 2023 08:19:03 +0000 Subject: [PATCH 17/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/test/test_geometry.py | 5 ++--- datamodel/test/test_views.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/datamodel/test/test_geometry.py b/datamodel/test/test_geometry.py index 1bc072ffe..f04b98b88 100644 --- a/datamodel/test/test_geometry.py +++ b/datamodel/test/test_geometry.py @@ -632,7 +632,7 @@ def test_cover_geometry_sync_on_update(self): == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" ) assert new_row["level"] is None - + def test_vw_tww_additional_ws_geometry_insert(self): # 1. insert geometry and no co_level and no wn_bottom_level # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1001', '1337_1001'); @@ -719,7 +719,7 @@ def test_vw_tww_additional_ws_geometry_insert(self): row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000006940" ) - + def test_vw_tww_additional_ws_geometry_update(self): # first insert # insert geometry with no co_level and no wn_bottom_level @@ -811,6 +811,5 @@ def test_vw_tww_additional_ws_geometry_update(self): ) - if __name__ == "__main__": unittest.main() diff --git a/datamodel/test/test_views.py b/datamodel/test/test_views.py index adedd5ce5..35c154db5 100644 --- a/datamodel/test/test_views.py +++ b/datamodel/test/test_views.py @@ -103,7 +103,7 @@ def test_vw_tww_wastewater_structure(self): row = cur.fetchone() assert row["backflow_level_current"] == decimal.Decimal("100.000") - + def test_vw_tww_additional_ws(self): row = { "identifier": "20", From 418b604e6258d6c2f88ce83e86c38a6b33ab5f74 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:40:44 +0100 Subject: [PATCH 18/48] set unique identifier --- datamodel/test/test_views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datamodel/test/test_views.py b/datamodel/test/test_views.py index adedd5ce5..f87c5653b 100644 --- a/datamodel/test/test_views.py +++ b/datamodel/test/test_views.py @@ -106,7 +106,7 @@ def test_vw_tww_wastewater_structure(self): def test_vw_tww_additional_ws(self): row = { - "identifier": "20", + "identifier": "201_created", "ws_type": "drainless_toilet", "situation3d_geometry": self.execute( "ST_SetSRID(ST_GeomFromText('POINT(2600000 1200000)'), 2056)" @@ -123,13 +123,13 @@ def test_vw_tww_additional_ws(self): obj_id = self.insert_check("vw_tww_additional_ws", row, expected_row) row = { - "identifier": "10", + "identifier": "202_accepted", "ws_type": "wwtp_structure", "co_material": 233, "wt_kind": 331, } - self.update_check("vw_tww_wastewater_structure", row, obj_id) + self.update_check("vw_tww_additional_ws", row, obj_id) cur = self.cursor() From ff6540f472241f159057d0916ad03b624cedef61 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:47:47 +0100 Subject: [PATCH 19/48] alter identifiers in test_geometry --- datamodel/test/test_geometry.py | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/datamodel/test/test_geometry.py b/datamodel/test/test_geometry.py index f04b98b88..97467dc14 100644 --- a/datamodel/test/test_geometry.py +++ b/datamodel/test/test_geometry.py @@ -635,11 +635,11 @@ def test_cover_geometry_sync_on_update(self): def test_vw_tww_additional_ws_geometry_insert(self): # 1. insert geometry and no co_level and no wn_bottom_level - # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1001', '1337_1001'); + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1338_1002', '1338_1002'); row = { "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", - "wn_obj_id": "1337_1001", - "co_obj_id": "1337_1001", + "wn_obj_id": "1338_1002", + "co_obj_id": "1338_1002", } expected_row = copy.deepcopy(row) # wastewaterstructure has the geometry but NaN as Z because of no co_level (geometry of cover): ST_SetSRID(ST_Collect(ST_MakePoint(2600000, 1200000, 'NaN')), 2056) @@ -650,23 +650,23 @@ def test_vw_tww_additional_ws_geometry_insert(self): expected_row["wn_bottom_level"] = None self.insert_check("vw_tww_additional_ws", row, expected_row) # cover geometry has the geometry but NaN as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 'NaN'), 2056) - row = self.select("cover", "1337_1001", schema="tww_od") + row = self.select("cover", "1338_1002", schema="tww_od") assert ( row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" ) # wastewater_node has the geometry but not 3d: ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056) - row = self.select("wastewater_node", "1337_1001", schema="tww_od") + row = self.select("wastewater_node", "1338_1002", schema="tww_od") assert row["situation3d_geometry"] == self.execute( "ST_SetSRID( ST_MakePoint(2600000, 1200000, 'NaN'), 2056)" ) # 2. insert geometry with no co_level and WITH wn_bottom_level - # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id, wn_bottom_level) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1002', '1337_1002', 200.000); + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id, wn_bottom_level) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1338_1003', '1338_1003', 200.000); row = { "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", - "wn_obj_id": "1337_1002", - "co_obj_id": "1337_1002", + "wn_obj_id": "1338_1003", + "co_obj_id": "1338_1003", "wn_bottom_level": "200.000", } expected_row = copy.deepcopy(row) @@ -678,24 +678,24 @@ def test_vw_tww_additional_ws_geometry_insert(self): expected_row["wn_bottom_level"] = "200.000" self.insert_check("vw_tww_additional_ws", row, expected_row) # cover geometry has the geometry but NaN as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 'NaN'), 2056) - row = self.select("cover", "1337_1002", schema="tww_od") + row = self.select("cover", "1338_1003", schema="tww_od") assert ( row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" ) # wastewater_node has the geometry and wn_buttom_level as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) - row = self.select("wastewater_node", "1337_1002", schema="tww_od") + row = self.select("wastewater_node", "1338_1003", schema="tww_od") assert ( row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000006940" ) # 3. insert geometry with Z and WITH co_level and WITH wn_bottom_level - # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id, wn_bottom_level, co_level) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1003', '1337_1003', 200.000, 500.000); + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id, wn_bottom_level, co_level) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1338_1004', '1338_1004', 200.000, 500.000); row = { "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", - "wn_obj_id": "1337_1003", - "co_obj_id": "1337_1003", + "wn_obj_id": "1338_1004", + "co_obj_id": "1338_1004", "wn_bottom_level": "200.000", "co_level": "500.000", } @@ -708,13 +708,13 @@ def test_vw_tww_additional_ws_geometry_insert(self): expected_row["wn_bottom_level"] = "200.000" self.insert_check("vw_tww_additional_ws", row, expected_row) # cover geometry has the geometry and co_level as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 500), 2056) - row = self.select("cover", "1337_1003", schema="tww_od") + row = self.select("cover", "1338_1004", schema="tww_od") assert ( row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000407F40" ) # wastewater_node has the geometry and wn_buttom_level as Z: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) - row = self.select("wastewater_node", "1337_1003", schema="tww_od") + row = self.select("wastewater_node", "1338_1004", schema="tww_od") assert ( row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000006940" @@ -723,12 +723,12 @@ def test_vw_tww_additional_ws_geometry_insert(self): def test_vw_tww_additional_ws_geometry_update(self): # first insert # insert geometry with no co_level and no wn_bottom_level - # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1337_1010', '1337_1010'); + # INSERT INTO tww_app.vw_tww_additional_ws (situation3d_geometry, wn_obj_id, co_obj_id) VALUES (ST_SetSRID(ST_MakePoint(2600000, 1200000), 2056), '1338_1010', '1338_1010'); row = { "situation3d_geometry": "0101000020080800000000000020D6434100000000804F3241", "ws_type": "manhole", - "wn_obj_id": "1337_1010", - "co_obj_id": "1337_1010", + "wn_obj_id": "1338_1010", + "co_obj_id": "1338_1010", } obj_id = self.insert("vw_tww_additional_ws", row) @@ -746,13 +746,13 @@ def test_vw_tww_additional_ws_geometry_update(self): # wn_bottom_level is new wn_bottom_level self.assertEqual(new_row["wn_bottom_level"], 200.000) # no change on cover geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000, 'NaN'), 2056) - new_row = self.select("cover", "1337_1010", schema="tww_od") + new_row = self.select("cover", "1338_1010", schema="tww_od") assert ( new_row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F3241000000000000F87F" ) # wastewater_node geometry has Z from new wn_bottom_level: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) - new_row = self.select("wastewater_node", "1337_1010", schema="tww_od") + new_row = self.select("wastewater_node", "1338_1010", schema="tww_od") assert ( new_row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000006940" @@ -772,13 +772,13 @@ def test_vw_tww_additional_ws_geometry_update(self): # no change on wn_bottom_level assert new_row["wn_bottom_level"] == 200.000 # cover geometry has Z from new co_level: ST_SetSRID(ST_MakePoint(2600000, 1200000, 500), 2056) - new_row = self.select("cover", "1337_1010", schema="tww_od") + new_row = self.select("cover", "1338_1010", schema="tww_od") assert ( new_row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000407F40" ) # no change on wastewater_node geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000, 200), 2056) - new_row = self.select("wastewater_node", "1337_1010", schema="tww_od") + new_row = self.select("wastewater_node", "1338_1010", schema="tww_od") assert ( new_row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000006940" @@ -798,13 +798,13 @@ def test_vw_tww_additional_ws_geometry_update(self): # wn_bottom_level is new wn_bottom_level assert new_row["wn_bottom_level"] == 300.000 # cover geometry has Z from new co_level: ST_SetSRID(ST_MakePoint(2600000, 1200000, 600), 2056) - new_row = self.select("cover", "1337_1010", schema="tww_od") + new_row = self.select("cover", "1338_1010", schema="tww_od") assert ( new_row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000C08240" ) # no change on wastewater_node geometry: ST_SetSRID(ST_MakePoint(2600000, 1200000, 300), 2056) - new_row = self.select("wastewater_node", "1337_1010", schema="tww_od") + new_row = self.select("wastewater_node", "1338_1010", schema="tww_od") assert ( new_row["situation3d_geometry"] == "01010000A0080800000000000020D6434100000000804F32410000000000C07240" From 43be36f1c39086fa5e7856b31808be4e8d1a1e19 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Jan 2024 10:00:03 +0100 Subject: [PATCH 20/48] Make update more dynamic In order to alter a ws from a "normal" to an "additional" ws, one can now simply update the ws_type to the corresponding table --- datamodel/app/view/vw_tww_additional_ws.py | 21 ++++++++-------- .../app/view/vw_tww_wastewater_structure.py | 25 ++++++++----------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index c697335f7..73237a9f8 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -343,18 +343,17 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): {update_wn} IF OLD.ws_type <> NEW.ws_type THEN - CASE - WHEN OLD.ws_type = 'wwtp_structure' THEN DELETE FROM tww_od.wwtp_structure WHERE obj_id = OLD.obj_id; - WHEN OLD.ws_type = 'small_treatment_plant' THEN DELETE FROM tww_od.small_treatment_plant WHERE obj_id = OLD.obj_id; - WHEN OLD.ws_type = 'drainless_toilet' THEN DELETE FROM tww_od.drainless_toilet WHERE obj_id = OLD.obj_id; - ELSE -- do nothing + CASE WHEN OLD.ws_type <> 'unknown' THEN + EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %I',OLD.ws_type,OLD.obj_id); END CASE; - - CASE - WHEN NEW.ws_type = 'wwtp_structure' THEN INSERT INTO tww_od.wwtp_structure (obj_id) VALUES(OLD.obj_id); - WHEN NEW.ws_type = 'small_treatment_plant' THEN INSERT INTO tww_od.small_treatment_plant (obj_id) VALUES(OLD.obj_id); - WHEN NEW.ws_type = 'drainless_toilet' THEN INSERT INTO tww_od.drainless_toilet (obj_id) VALUES(OLD.obj_id); - ELSE -- do nothing + + CASE WHEN NEW.ws_type <> 'unknown' THEN + EXECUTE FORMAT('INSERT INTO FROM tww_od.%I(obj_id) VALUES (%I)',NEW.ws_type,OLD.obj_id); + EXCEPTION + WHEN undefined_table THEN + RAISE NOTICE 'table tww_od.% does not exist, rolling back',NEW.ws_type; + RETURN OLD; + END; END CASE; END IF; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 1fd0292ef..4aee32a2f 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -257,7 +257,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit {insert_ii} ELSE - RAISE NOTICE 'Wastewater structure type not known (%)', NEW.ws_type; -- ERROR + RAISE NOTICE 'Wastewater structure type not handled by this view (%)', NEW.ws_type; -- ERROR END CASE; {insert_wn} @@ -402,20 +402,17 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit {update_wn} IF OLD.ws_type <> NEW.ws_type THEN - CASE - WHEN OLD.ws_type = 'manhole' THEN DELETE FROM tww_od.manhole WHERE obj_id = OLD.obj_id; - WHEN OLD.ws_type = 'special_structure' THEN DELETE FROM tww_od.special_structure WHERE obj_id = OLD.obj_id; - WHEN OLD.ws_type = 'discharge_point' THEN DELETE FROM tww_od.discharge_point WHERE obj_id = OLD.obj_id; - WHEN OLD.ws_type = 'infiltration_installation' THEN DELETE FROM tww_od.infiltration_installation WHERE obj_id = OLD.obj_id; - ELSE -- do nothing + CASE WHEN OLD.ws_type <> 'unknown' THEN + EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %I',OLD.ws_type,OLD.obj_id); END CASE; - - CASE - WHEN NEW.ws_type = 'manhole' THEN INSERT INTO tww_od.manhole (obj_id) VALUES(OLD.obj_id); - WHEN NEW.ws_type = 'special_structure' THEN INSERT INTO tww_od.special_structure (obj_id) VALUES(OLD.obj_id); - WHEN NEW.ws_type = 'discharge_point' THEN INSERT INTO tww_od.discharge_point (obj_id) VALUES(OLD.obj_id); - WHEN NEW.ws_type = 'infiltration_installation' THEN INSERT INTO tww_od.infiltration_installation (obj_id) VALUES(OLD.obj_id); - ELSE -- do nothing + + CASE WHEN NEW.ws_type <> 'unknown' THEN + EXECUTE FORMAT('INSERT INTO FROM tww_od.%I(obj_id) VALUES (%I)',NEW.ws_type,OLD.obj_id); + EXCEPTION + WHEN undefined_table THEN + RAISE NOTICE 'table tww_od.% does not exist, rolling back',NEW.ws_type; + RETURN OLD; + END; END CASE; END IF; From 261b34cd64b21f0360748ad27790351c4b01ae0b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 09:00:16 +0000 Subject: [PATCH 21/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/vw_tww_additional_ws.py | 2 +- datamodel/app/view/vw_tww_wastewater_structure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 73237a9f8..96c2f1bdf 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -346,7 +346,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): CASE WHEN OLD.ws_type <> 'unknown' THEN EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %I',OLD.ws_type,OLD.obj_id); END CASE; - + CASE WHEN NEW.ws_type <> 'unknown' THEN EXECUTE FORMAT('INSERT INTO FROM tww_od.%I(obj_id) VALUES (%I)',NEW.ws_type,OLD.obj_id); EXCEPTION diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 4aee32a2f..2459fb3ab 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -405,7 +405,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit CASE WHEN OLD.ws_type <> 'unknown' THEN EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %I',OLD.ws_type,OLD.obj_id); END CASE; - + CASE WHEN NEW.ws_type <> 'unknown' THEN EXECUTE FORMAT('INSERT INTO FROM tww_od.%I(obj_id) VALUES (%I)',NEW.ws_type,OLD.obj_id); EXCEPTION From 4035fc82ceb8ae1232fac9735520b315c22fe82c Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Jan 2024 12:00:35 +0100 Subject: [PATCH 22/48] escape % with %% for update --- datamodel/app/view/vw_tww_additional_ws.py | 6 +++--- datamodel/app/view/vw_tww_wastewater_structure.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 96c2f1bdf..1775db77c 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -344,14 +344,14 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %I',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%I',OLD.ws_type,OLD.obj_id); END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%I(obj_id) VALUES (%I)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); EXCEPTION WHEN undefined_table THEN - RAISE NOTICE 'table tww_od.% does not exist, rolling back',NEW.ws_type; + RAISE NOTICE 'table tww_od.%% does not exist, rolling back',NEW.ws_type; RETURN OLD; END; END CASE; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 2459fb3ab..c360d7b0a 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -403,14 +403,14 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %I',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%I',OLD.ws_type,OLD.obj_id); END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%I(obj_id) VALUES (%I)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); EXCEPTION WHEN undefined_table THEN - RAISE NOTICE 'table tww_od.% does not exist, rolling back',NEW.ws_type; + RAISE NOTICE 'table tww_od.%% does not exist, rolling back',NEW.ws_type; RETURN OLD; END; END CASE; From ad8b3c75629d1de2d6fa98f457da7b5d42bc0971 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Jan 2024 12:56:28 +0100 Subject: [PATCH 23/48] Syntax corrections --- datamodel/app/view/vw_tww_additional_ws.py | 1 - datamodel/app/view/vw_tww_wastewater_structure.py | 1 - 2 files changed, 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 1775db77c..52b1f1483 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -353,7 +353,6 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): WHEN undefined_table THEN RAISE NOTICE 'table tww_od.%% does not exist, rolling back',NEW.ws_type; RETURN OLD; - END; END CASE; END IF; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index c360d7b0a..eb8011877 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -412,7 +412,6 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit WHEN undefined_table THEN RAISE NOTICE 'table tww_od.%% does not exist, rolling back',NEW.ws_type; RETURN OLD; - END; END CASE; END IF; From 5c93c3bf315fee3456c7a47963851cfcb0a59879 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Jan 2024 13:55:36 +0100 Subject: [PATCH 24/48] wrap exception in Begin/end --- datamodel/app/view/vw_tww_additional_ws.py | 11 ++++++----- datamodel/app/view/vw_tww_wastewater_structure.py | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 52b1f1483..1a8240db6 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -348,11 +348,12 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); - EXCEPTION - WHEN undefined_table THEN - RAISE NOTICE 'table tww_od.%% does not exist, rolling back',NEW.ws_type; - RETURN OLD; + BEGIN + EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); + EXCEPTION + WHEN undefined_table THEN + RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; + END; END CASE; END IF; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index eb8011877..570ba4471 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -407,11 +407,12 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); - EXCEPTION - WHEN undefined_table THEN - RAISE NOTICE 'table tww_od.%% does not exist, rolling back',NEW.ws_type; - RETURN OLD; + BEGIN + EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); + EXCEPTION + WHEN undefined_table THEN + RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; + END; END CASE; END IF; From 5a09a79917f37caf9c23ab2b196139b835f28212 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:34:32 +0100 Subject: [PATCH 25/48] pass obj_id as string --- datamodel/app/view/vw_tww_additional_ws.py | 4 ++-- datamodel/app/view/vw_tww_wastewater_structure.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 1a8240db6..30aaeea6b 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -344,12 +344,12 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%I',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%s',OLD.ws_type,OLD.obj_id); END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%s)',NEW.ws_type,OLD.obj_id); EXCEPTION WHEN undefined_table THEN RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 570ba4471..bb9517d8f 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -403,12 +403,12 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%I',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%s',OLD.ws_type,OLD.obj_id); END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%I)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%s)',NEW.ws_type,OLD.obj_id); EXCEPTION WHEN undefined_table THEN RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; From c395212d1a0d06cb3fb2cbdcac2200bc68cbed1d Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:54:18 +0100 Subject: [PATCH 26/48] Literal not String --- datamodel/app/view/vw_tww_additional_ws.py | 4 ++-- datamodel/app/view/vw_tww_wastewater_structure.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 30aaeea6b..b37237278 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -344,12 +344,12 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%s',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%s)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); EXCEPTION WHEN undefined_table THEN RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index bb9517d8f..f5a5e24bb 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -403,12 +403,12 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%s',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); END CASE; CASE WHEN NEW.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('INSERT INTO FROM tww_od.%%I(obj_id) VALUES (%%s)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); EXCEPTION WHEN undefined_table THEN RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; From de619a0e3eaffecc9c4eba0fc51b0dfcef586cd0 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:57:36 +0200 Subject: [PATCH 27/48] remove changes in create_views and vw_tww_import to allow syncing the fork --- datamodel/app/view/create_views.py | 1 - datamodel/app/view/vw_tww_import.sql | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index ab7c584a0..e8bb78db0 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -195,7 +195,6 @@ def create_views( srid, pg_service=pg_service, extra_definition=tww_wastewater_structure_extra ) vw_tww_reach(pg_service=pg_service, extra_definition=tww_reach_extra) - vw_tww_additional_ws(srid, pg_service=pg_service) run_sql("app/view/vw_file.sql", pg_service, variables) MultipleInheritance( diff --git a/datamodel/app/view/vw_tww_import.sql b/datamodel/app/view/vw_tww_import.sql index a0255eeaf..5172cdb75 100644 --- a/datamodel/app/view/vw_tww_import.sql +++ b/datamodel/app/view/vw_tww_import.sql @@ -3,14 +3,14 @@ CREATE OR REPLACE VIEW tww_app.import_vw_manhole AS SELECT DISTINCT ON (ws.obj_id) ws.obj_id, ws.identifier, - COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry)::geometry(POINTZ, %(SRID)s) AS situation_geometry, + COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry)::geometry(POINTZ, {SRID}) AS situation_geometry, main_co.cover_shape as co_shape, main_co.diameter as co_diameter, main_co.material as co_material, main_co.positional_accuracy as co_positional_accuracy, main_co.level as co_level, ws._depth::numeric(6, 3) AS _depth, - ws._usage_current AS _channel_usage_current, + wn._usage_current AS _channel_usage_current, ma.material as ma_material, ma.dimension1 as ma_dimension1, ma.dimension2 as ma_dimension2, @@ -317,11 +317,11 @@ BEGIN -- catch EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'EXCEPTION: %%%%', SQLERRM; + RAISE NOTICE 'EXCEPTION: %%', SQLERRM; RETURN NEW; END; $BODY$ LANGUAGE plpgsql; -$TRIGGER$, %(SRID)s); +$TRIGGER$, {SRID}); END $DO$; @@ -334,7 +334,7 @@ CREATE TRIGGER after_update_try_structure_update WHEN ( ( NEW.structure_okay IS NOT TRUE ) AND NOT( OLD.inlet_okay IS NOT TRUE AND NEW.inlet_okay IS TRUE ) AND NOT( OLD.outlet_okay IS NOT TRUE AND NEW.outlet_okay IS TRUE ) ) - EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update(%(SRID)s); + EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update({SRID}); DROP TRIGGER IF EXISTS after_insert_try_structure_update ON tww_od.import_manhole_quarantine; @@ -342,7 +342,7 @@ CREATE TRIGGER after_insert_try_structure_update AFTER INSERT ON tww_od.import_manhole_quarantine FOR EACH ROW - EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update(%(SRID)s); + EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update({SRID}); -- Some information: -- 1. new lets 0 - old lets 0 -> do nothing @@ -386,14 +386,14 @@ BEGIN -- handle inlets IF ( new_lets > 1 AND old_lets > 0 ) OR old_lets > 1 THEN -- request for update because new lets are bigger 1 (and old lets not 0 ) or old lets are bigger 1 - RAISE NOTICE 'Impossible to assign %%s - manual edit needed.', let_kind; + RAISE NOTICE 'Impossible to assign %s - manual edit needed.', let_kind; ELSE IF new_lets = 0 AND old_lets > 0 THEN -- request for delete because no new lets but old lets - RAISE NOTICE 'No new %%s but old ones - manual delete needed.', let_kind; + RAISE NOTICE 'No new %s but old ones - manual delete needed.', let_kind; ELSIF new_lets > 0 AND old_lets = 0 THEN -- request for create because no old lets but new lets - RAISE NOTICE 'No old %%s but new ones - manual create needed.', let_kind; + RAISE NOTICE 'No old %s but new ones - manual create needed.', let_kind; ELSE IF new_lets = 1 AND old_lets = 1 THEN IF let_kind='inlet' THEN @@ -440,10 +440,10 @@ BEGIN WHERE ws.obj_id = NEW.obj_id ); END IF; - RAISE NOTICE '%%s updated', let_kind; + RAISE NOTICE '%s updated', let_kind; ELSE -- do nothing - RAISE NOTICE 'No %%s - nothing to do', let_kind; + RAISE NOTICE 'No %s - nothing to do', let_kind; END IF; IF let_kind='inlet' THEN @@ -464,7 +464,7 @@ BEGIN -- catch EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'EXCEPTION: %%', SQLERRM; + RAISE NOTICE 'EXCEPTION: %', SQLERRM; RETURN NEW; END; $BODY$ LANGUAGE plpgsql; @@ -524,4 +524,4 @@ CREATE TRIGGER after_mutation_delete_when_okay ON tww_od.import_manhole_quarantine FOR EACH ROW WHEN ( NEW.structure_okay IS TRUE AND NEW.inlet_okay IS TRUE AND NEW.outlet_okay IS TRUE ) - EXECUTE PROCEDURE tww_app.import_manhole_quarantine_delete_entry(); + EXECUTE PROCEDURE tww_app.import_manhole_quarantine_delete_entry(); \ No newline at end of file From d37108e8658f0fb422f1d925f78da40a2ca46fde Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:01:41 +0200 Subject: [PATCH 28/48] remove changes in change_views V2 --- datamodel/app/view/create_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index e8bb78db0..3fe7ea050 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -4,7 +4,6 @@ import psycopg2 from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance -from vw_tww_additional_ws import vw_tww_additional_ws from vw_tww_reach import vw_tww_reach from vw_tww_wastewater_structure import vw_tww_wastewater_structure from yaml import safe_load From 49f0faab0517b618f96c766cb4d56d3572d2d737 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:12:35 +0200 Subject: [PATCH 29/48] Next try on create_views --- datamodel/app/view/create_views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index 3fe7ea050..c330630e2 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -194,6 +194,7 @@ def create_views( srid, pg_service=pg_service, extra_definition=tww_wastewater_structure_extra ) vw_tww_reach(pg_service=pg_service, extra_definition=tww_reach_extra) + run_sql("app/view/vw_file.sql", pg_service, variables) MultipleInheritance( From 65f5d6f2cddb5ec1ea8ca45f9c5ea7177c44419f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:12:52 +0000 Subject: [PATCH 30/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/vw_tww_import.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamodel/app/view/vw_tww_import.sql b/datamodel/app/view/vw_tww_import.sql index 5172cdb75..b29e4f54d 100644 --- a/datamodel/app/view/vw_tww_import.sql +++ b/datamodel/app/view/vw_tww_import.sql @@ -524,4 +524,4 @@ CREATE TRIGGER after_mutation_delete_when_okay ON tww_od.import_manhole_quarantine FOR EACH ROW WHEN ( NEW.structure_okay IS TRUE AND NEW.inlet_okay IS TRUE AND NEW.outlet_okay IS TRUE ) - EXECUTE PROCEDURE tww_app.import_manhole_quarantine_delete_entry(); \ No newline at end of file + EXECUTE PROCEDURE tww_app.import_manhole_quarantine_delete_entry(); From 6dcb1de9f311f18464c013f32d5ef4418f640a32 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:16:19 +0200 Subject: [PATCH 31/48] load view in create_app --- datamodel/app/create_app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datamodel/app/create_app.py b/datamodel/app/create_app.py index a9bd8928d..676a92f48 100755 --- a/datamodel/app/create_app.py +++ b/datamodel/app/create_app.py @@ -12,6 +12,7 @@ from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance from view.vw_tww_reach import vw_tww_reach from view.vw_tww_wastewater_structure import vw_tww_wastewater_structure +from view.vw_tww_additional_ws import vw_tww_additional_ws from yaml import safe_load @@ -143,6 +144,7 @@ def create_app( srid, pg_service=pg_service, extra_definition=tww_wastewater_structure_extra ) vw_tww_reach(pg_service=pg_service, extra_definition=tww_reach_extra) + vw_tww_additional_ws(srid, pg_service=pg_service) run_sql_file("view/vw_file.sql", pg_service, variables) From 3676887854627e24d175ff88ccfeb34814385269 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:16:36 +0000 Subject: [PATCH 32/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/create_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamodel/app/create_app.py b/datamodel/app/create_app.py index 676a92f48..903957602 100755 --- a/datamodel/app/create_app.py +++ b/datamodel/app/create_app.py @@ -10,9 +10,9 @@ import psycopg2 as psycopg from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance +from view.vw_tww_additional_ws import vw_tww_additional_ws from view.vw_tww_reach import vw_tww_reach from view.vw_tww_wastewater_structure import vw_tww_wastewater_structure -from view.vw_tww_additional_ws import vw_tww_additional_ws from yaml import safe_load From 6f5ec5c7967da2734d877a8ee8584479d0f63ae7 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:19:55 +0200 Subject: [PATCH 33/48] switsch from psycopg2 to psycopg --- datamodel/app/view/vw_tww_additional_ws.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index b37237278..e2a686055 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -5,7 +5,10 @@ import argparse import os -import psycopg2 +try: + import psycopg +except ImportError: + import psycopg2 as psycopg from pirogue.utils import insert_command, select_columns, update_command @@ -21,7 +24,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): variables = {"SRID": int(srid)} - conn = psycopg2.connect(f"service={pg_service}") + conn = psycopg.connect(f"service={pg_service}") cursor = conn.cursor() view_sql = """ From 08aecca4f9db744675813dcfa18f4dfed69e4ede Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:43:36 +0200 Subject: [PATCH 34/48] alter handling of type changes --- datamodel/app/view/vw_tww_wastewater_structure.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 147b6f743..e0f66ff97 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -405,12 +405,9 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); END CASE; - CASE WHEN NEW.ws_type <> 'unknown' THEN + CASE WHEN NEW.ws_type = ANY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant'] THEN BEGIN EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); - EXCEPTION - WHEN undefined_table THEN - RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; END; END CASE; END IF; From 4b5f581b55ed026c93582a90dd286750f2605eac Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:49:35 +0200 Subject: [PATCH 35/48] add array --- datamodel/app/view/vw_tww_wastewater_structure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index e0f66ff97..2351f3513 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -405,7 +405,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); END CASE; - CASE WHEN NEW.ws_type = ANY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant'] THEN + CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); END; From 4c03c78029a4d07131acfd8e0a522aef36b2f6bc Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:54:57 +0200 Subject: [PATCH 36/48] remove _usage_current and _function_hierarchic --- datamodel/app/view/vw_tww_additional_ws.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index e2a686055..c6e8bb481 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -485,8 +485,6 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): skip_columns=[ "detail_geometry3d_geometry", "last_modification", - "_usage_current", - "_function_hierarchic", "_label", "_cover_label", "_bottom_label", From 867f34e7dbcac3c9a07fcb73210ac5ac4590fd8a Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:34:34 +0200 Subject: [PATCH 37/48] remove more nonexistent fields --- datamodel/app/view/vw_tww_additional_ws.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index c6e8bb481..231ada921 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -108,8 +108,6 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): "_bottom_label", "_input_label", "_output_label", - "_usage_current", - "_function_hierarchic", "fk_main_cover", "fk_main_wastewater_node", "detail_geometry3d_geometry", @@ -248,8 +246,6 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): "_bottom_label", "_input_label", "_output_label", - "_usage_current", - "_function_hierarchic", "fk_main_cover", "fk_main_wastewater_node", "detail_geometry3d_geometry", From 13454c0cccf40442d5cee5086137f021a0ddbf82 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:56:11 +0200 Subject: [PATCH 38/48] _usage_current the next --- datamodel/app/view/vw_tww_additional_ws.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 231ada921..5fe534c96 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -69,8 +69,8 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): , ws._bottom_label , ws._input_label , ws._output_label - , ws._usage_current AS _channel_usage_current - , ws._function_hierarchic AS _channel_function_hierarchic + , wn._usage_current AS _channel_usage_current + , wn._function_hierarchic AS _channel_function_hierarchic FROM tww_od.wastewater_structure ws LEFT JOIN tww_od.cover main_co ON main_co.obj_id = ws.fk_main_cover From ee0294391b07b83f15756d1fb6ce480bb28d6dc5 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:09:02 +0200 Subject: [PATCH 39/48] alter srid handling --- datamodel/app/view/vw_tww_additional_ws.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 5fe534c96..548d8210f 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -53,7 +53,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): , main_co_sp.renovation_demand AS co_renovation_demand , {main_co_cols} - , ST_Force2D(COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry))::geometry(Point, %(SRID)s) AS situation3d_geometry + , ST_Force2D(COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry))::geometry(Point, {srid}) AS situation3d_geometry , {wt_columns} @@ -92,6 +92,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): ALTER VIEW tww_app.vw_tww_additional_ws ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); ALTER VIEW tww_app.vw_tww_additional_ws ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); """.format( + srid=srid, ws_cols=select_columns( pg_cur=cursor, table_schema="tww_od", @@ -346,12 +347,9 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); END CASE; - CASE WHEN NEW.ws_type <> 'unknown' THEN + CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); - EXCEPTION - WHEN undefined_table THEN - RAISE NOTICE 'table tww_od.%% does not exist',NEW.ws_type; END; END CASE; END IF; @@ -380,7 +378,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): SET situation3d_geometry = ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )), - ST_Z(WN.situation3d_geometry)), %(SRID)s ) + ST_Z(WN.situation3d_geometry)), {srid} ) WHERE obj_id IN ( SELECT obj_id FROM tww_od.wastewater_networkelement @@ -392,7 +390,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): SET situation3d_geometry = ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )), - ST_Z(CO.situation3d_geometry)), %(SRID)s ) + ST_Z(CO.situation3d_geometry)), {srid} ) WHERE obj_id IN ( SELECT obj_id FROM tww_od.structure_part @@ -408,7 +406,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )), - ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s ) + ST_Z(ST_PointN(RE.progression3d_geometry, 1))), {srid} ) ) ) WHERE fk_reach_point_from IN ( @@ -425,7 +423,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )), - ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s ) + ST_Z(ST_PointN(RE.progression3d_geometry, 1))), {srid} ) ) ) WHERE fk_reach_point_to IN ( @@ -447,6 +445,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): CREATE TRIGGER vw_tww_additional_ws_UPDATE INSTEAD OF UPDATE ON tww_app.vw_tww_additional_ws FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_additional_ws_UPDATE(); """.format( + srid=srid, update_co=update_command( pg_cur=cursor, table_schema="tww_od", From c30695bb271a617a65f67cab3ce42542e3f0611f Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:40:01 +0200 Subject: [PATCH 40/48] wrap execute format in begin/end --- datamodel/app/view/vw_tww_additional_ws.py | 4 +++- datamodel/app/view/vw_tww_wastewater_structure.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 548d8210f..0084b5f5e 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -344,7 +344,9 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + BEGIN + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 2351f3513..25b2ed20b 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -402,7 +402,9 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + BEGIN + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN From da34826640cc81dfac0ac8a9195b9ee7f073e7a5 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:45:21 +0200 Subject: [PATCH 41/48] alter %% to % --- datamodel/app/view/vw_tww_additional_ws.py | 4 ++-- datamodel/app/view/vw_tww_wastewater_structure.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 0084b5f5e..55ef4b9c6 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -345,13 +345,13 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); END; END CASE; END IF; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 25b2ed20b..a3517b682 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -403,13 +403,13 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); END; END CASE; END IF; From 695c9d72e70f3f2ad1aa4f4b0cf41606a83f8d31 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:06:09 +0200 Subject: [PATCH 42/48] rollback --- datamodel/app/view/vw_tww_additional_ws.py | 4 ++-- datamodel/app/view/vw_tww_wastewater_structure.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 55ef4b9c6..0084b5f5e 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -345,13 +345,13 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); END; END CASE; END IF; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index a3517b682..25b2ed20b 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -403,13 +403,13 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); END; END CASE; END IF; From 191fba1c345b56eabaed72b9c9d5761fcab49115 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:20:23 +0200 Subject: [PATCH 43/48] next try %% to % --- datamodel/app/view/vw_tww_additional_ws.py | 4 ++-- datamodel/app/view/vw_tww_import.sql | 2 +- datamodel/app/view/vw_tww_wastewater_structure.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 0084b5f5e..55ef4b9c6 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -345,13 +345,13 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); END; END CASE; END IF; diff --git a/datamodel/app/view/vw_tww_import.sql b/datamodel/app/view/vw_tww_import.sql index b29e4f54d..757a3f36f 100644 --- a/datamodel/app/view/vw_tww_import.sql +++ b/datamodel/app/view/vw_tww_import.sql @@ -317,7 +317,7 @@ BEGIN -- catch EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'EXCEPTION: %%', SQLERRM; + RAISE NOTICE 'EXCEPTION: %', SQLERRM; RETURN NEW; END; $BODY$ LANGUAGE plpgsql; diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 25b2ed20b..a3517b682 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -403,13 +403,13 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%%I WHERE obj_id = %%L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%%I(obj_id) VALUES (%%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); END; END CASE; END IF; From a88e77a299e719fdd3429372b72133773d64ca7e Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:33:14 +0200 Subject: [PATCH 44/48] trying another solution --- datamodel/app/view/vw_tww_additional_ws.py | 6 ++++-- datamodel/app/view/vw_tww_wastewater_structure.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index 55ef4b9c6..c6da35aeb 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -345,13 +345,13 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT({literal_delete_on_ws_change}); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT({literal_insert_on_ws_change}); END; END CASE; END IF; @@ -448,6 +448,8 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_additional_ws_UPDATE(); """.format( srid=srid, + literal_delete_on_ws_change ="'DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id", + literal_insert_on_ws_change ="'INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id", update_co=update_command( pg_cur=cursor, table_schema="tww_od", diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index a3517b682..8e9f64f41 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -403,13 +403,13 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit IF OLD.ws_type <> NEW.ws_type THEN CASE WHEN OLD.ws_type <> 'unknown' THEN BEGIN - EXECUTE FORMAT('DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id); + EXECUTE FORMAT({literal_delete_on_ws_change}); END; END CASE; CASE WHEN NEW.ws_type = ANY(ARRAY['manhole','special_structure','discharge_point','infiltration_installation','drainless_toilet','wwtp_structure','small_treatment_plant']) THEN BEGIN - EXECUTE FORMAT('INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id); + EXECUTE FORMAT({literal_insert_on_ws_change}); END; END CASE; END IF; @@ -509,6 +509,8 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_wastewater_structure_UPDATE(); """.format( srid=srid, + literal_delete_on_ws_change ="'DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id", + literal_insert_on_ws_change ="'INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id", update_co=update_command( pg_cur=cursor, table_schema="tww_od", From b36ed4d9bef297efef6b396d2eb6bda61fd5baf7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 14:34:01 +0000 Subject: [PATCH 45/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datamodel/app/view/vw_tww_additional_ws.py | 4 ++-- datamodel/app/view/vw_tww_wastewater_structure.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index c6da35aeb..5cd5815f5 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -448,8 +448,8 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_additional_ws_UPDATE(); """.format( srid=srid, - literal_delete_on_ws_change ="'DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id", - literal_insert_on_ws_change ="'INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id", + literal_delete_on_ws_change="'DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id", + literal_insert_on_ws_change="'INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id", update_co=update_command( pg_cur=cursor, table_schema="tww_od", diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index 8e9f64f41..6fb1be293 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -509,8 +509,8 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_wastewater_structure_UPDATE(); """.format( srid=srid, - literal_delete_on_ws_change ="'DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id", - literal_insert_on_ws_change ="'INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id", + literal_delete_on_ws_change="'DELETE FROM tww_od.%I WHERE obj_id = %L',OLD.ws_type,OLD.obj_id", + literal_insert_on_ws_change="'INSERT INTO tww_od.%I(obj_id) VALUES (%L)',NEW.ws_type,OLD.obj_id", update_co=update_command( pg_cur=cursor, table_schema="tww_od", From a71c316657dda62d17c4f06b0f1af7ca9896b912 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:41:27 +0200 Subject: [PATCH 46/48] drop variables --- datamodel/app/view/vw_tww_additional_ws.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/datamodel/app/view/vw_tww_additional_ws.py b/datamodel/app/view/vw_tww_additional_ws.py index c6da35aeb..df201fa7a 100644 --- a/datamodel/app/view/vw_tww_additional_ws.py +++ b/datamodel/app/view/vw_tww_additional_ws.py @@ -22,8 +22,6 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): pg_service = os.getenv("PGSERVICE") assert pg_service - variables = {"SRID": int(srid)} - conn = psycopg.connect(f"service={pg_service}") cursor = conn.cursor() @@ -184,7 +182,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): ), ) - cursor.execute(view_sql, variables) + cursor.execute(view_sql) trigger_insert_sql = """ CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_additional_ws_INSERT() @@ -539,7 +537,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): ), ) - cursor.execute(update_trigger_sql, variables) + cursor.execute(update_trigger_sql) trigger_delete_sql = """ CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_additional_ws_DELETE() @@ -556,7 +554,7 @@ def vw_tww_additional_ws(srid: int, pg_service: str = None): CREATE TRIGGER vw_tww_additional_ws_DELETE INSTEAD OF DELETE ON tww_app.vw_tww_additional_ws FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_additional_ws_DELETE(); """ - cursor.execute(trigger_delete_sql, variables) + cursor.execute(trigger_delete_sql) extras = """ ALTER VIEW tww_app.vw_tww_additional_ws ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); From caf7445518b3a5fb36b03d665b5aae34f7cd3379 Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:52:37 +0200 Subject: [PATCH 47/48] rollback for plain sql file --- datamodel/app/view/vw_tww_import.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamodel/app/view/vw_tww_import.sql b/datamodel/app/view/vw_tww_import.sql index 757a3f36f..b29e4f54d 100644 --- a/datamodel/app/view/vw_tww_import.sql +++ b/datamodel/app/view/vw_tww_import.sql @@ -317,7 +317,7 @@ BEGIN -- catch EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'EXCEPTION: %', SQLERRM; + RAISE NOTICE 'EXCEPTION: %%', SQLERRM; RETURN NEW; END; $BODY$ LANGUAGE plpgsql; From cc7345279bdde9694449fee77e6b7ba0bfc5bf3c Mon Sep 17 00:00:00 2001 From: cymed <102035587+cymed@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:58:42 +0200 Subject: [PATCH 48/48] test psycopg3 compatibility --- datamodel/test/test_views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/datamodel/test/test_views.py b/datamodel/test/test_views.py index 524590d7a..fa710d78a 100644 --- a/datamodel/test/test_views.py +++ b/datamodel/test/test_views.py @@ -138,7 +138,11 @@ def test_vw_tww_additional_ws(self): self.update_check("vw_tww_additional_ws", row, obj_id) - cur = self.cursor() + try: + cur = self.cursor(row_factory=psycopg.rows.dict_row) + except AttributeError: + # remove when dropping psycopg2 support + cur = self.cursor(cursor_factory=psycopg_extras.DictCursor) cur.execute( "SELECT * FROM tww_od.wastewater_networkelement NE LEFT JOIN tww_od.wastewater_node NO ON NO.obj_id = NE.obj_id WHERE fk_wastewater_structure='{obj_id}' ".format(