From 5209895a959bab2ec6dd319479084b5f4ff0de52 Mon Sep 17 00:00:00 2001 From: Denis Rouzaud Date: Tue, 9 Apr 2024 07:38:36 +0200 Subject: [PATCH] upgrade to psycopg3 --- .github/workflows/plugin-test.yml | 3 ++ datamodel/.docker/Dockerfile | 3 +- .../app/swmm_views/04_vw_swmm_conduits.sql | 2 +- .../swmm_views/09_vw_swmm_subcatchments.sql | 2 +- .../app/swmm_views/12_vw_swmm_raingages.sql | 2 +- .../vw_catchment_area_connections.sql | 4 +- datamodel/app/view/create_views.py | 6 +-- datamodel/app/view/vw_change_points.sql | 2 +- datamodel/app/view/vw_oo_overflow.yaml | 2 +- datamodel/app/view/vw_tww_import.sql | 22 ++++---- datamodel/app/view/vw_tww_reach.py | 4 +- .../app/view/vw_tww_wastewater_structure.py | 25 ++++----- datamodel/requirements.txt | 4 +- datamodel/test/test_geometry.py | 5 +- datamodel/test/test_import.py | 28 +++++----- datamodel/test/test_label.py | 5 +- datamodel/test/test_network.py | 9 ++-- datamodel/test/test_on_delete.py | 5 +- datamodel/test/test_removed_fields.py | 7 ++- datamodel/test/test_schemas.py | 5 +- datamodel/test/test_swmm.py | 5 +- datamodel/test/test_symbology.py | 5 +- datamodel/test/test_triggers.py | 5 +- datamodel/test/test_views.py | 7 ++- datamodel/test/utils.py | 15 +++--- datamodel/update/post-all.py | 4 +- docs/en/installation-guide/requirements.rst | 2 +- .../interlis/interlis_importer_exporter.py | 8 +-- .../interlis/utils/various.py | 4 +- .../processing_provider/TwwSwmm.py | 54 +++++++++---------- 30 files changed, 122 insertions(+), 132 deletions(-) diff --git a/.github/workflows/plugin-test.yml b/.github/workflows/plugin-test.yml index 13617a1ef..0a7f65d14 100644 --- a/.github/workflows/plugin-test.yml +++ b/.github/workflows/plugin-test.yml @@ -40,6 +40,9 @@ jobs: with: submodules: recursive + - name: Setup env + run: cp .env.example .env + - name: Package PyPI Packages run: sudo ./plugin/scripts/package-pip-packages.sh diff --git a/datamodel/.docker/Dockerfile b/datamodel/.docker/Dockerfile index 867c49c12..731235563 100644 --- a/datamodel/.docker/Dockerfile +++ b/datamodel/.docker/Dockerfile @@ -27,7 +27,6 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip install -r datamodel/requirements.txt RUN if [ "${RUN_TEST}" = "True" ]; then pip install -r datamodel/requirements-test.txt; fi - # Configure the postgres connections RUN printf '[postgres]\ndbname=postgres\nuser=postgres\n' >> /etc/postgresql-common/pg_service.conf RUN printf '[pg_tww]\ndbname=tww\nuser=postgres\n' >> /etc/postgresql-common/pg_service.conf @@ -41,7 +40,7 @@ RUN if [ "${AUTO_INIT}" = "True" ]; then ln -s /src/datamodel/.docker/init_db.sh # Some defaults ENV POSTGRES_PASSWORD=postgres -# otherwise psycopg2 cannot connect +# otherwise psycopg cannot connect ENV PGSERVICEFILE=/etc/postgresql-common/pg_service.conf ENV PGSERVICE=pg_tww diff --git a/datamodel/app/swmm_views/04_vw_swmm_conduits.sql b/datamodel/app/swmm_views/04_vw_swmm_conduits.sql index 0ec5a92bd..e9c18900c 100644 --- a/datamodel/app/swmm_views/04_vw_swmm_conduits.sql +++ b/datamodel/app/swmm_views/04_vw_swmm_conduits.sql @@ -113,7 +113,7 @@ SELECT END ) as description, cfhy.value_en as tag, - ST_CurveToLine(st_force3d(progression3d_geometry))::geometry(LineStringZ, %(SRID)s) as geom, + ST_CurveToLine(st_force3d(progression3d_geometry))::geometry(LineStringZ, {SRID}) as geom, CASE WHEN status IN (7959, 6529, 6526) THEN 'planned' ELSE 'current' diff --git a/datamodel/app/swmm_views/09_vw_swmm_subcatchments.sql b/datamodel/app/swmm_views/09_vw_swmm_subcatchments.sql index a40bec8f1..b2e7c2227 100644 --- a/datamodel/app/swmm_views/09_vw_swmm_subcatchments.sql +++ b/datamodel/app/swmm_views/09_vw_swmm_subcatchments.sql @@ -69,7 +69,7 @@ SELECT THEN NULL END as description, ca.obj_id as tag, - ST_CurveToLine(perimeter_geometry)::geometry(Polygon, %(SRID)s) as geom, + ST_CurveToLine(perimeter_geometry)::geometry(Polygon, {SRID}) as geom, CASE WHEN state = 'rw_current' OR state = 'ww_current' THEN 'current' WHEN state = 'rw_planned' OR state = 'ww_planned' THEN 'planned' diff --git a/datamodel/app/swmm_views/12_vw_swmm_raingages.sql b/datamodel/app/swmm_views/12_vw_swmm_raingages.sql index a172915a6..682366877 100644 --- a/datamodel/app/swmm_views/12_vw_swmm_raingages.sql +++ b/datamodel/app/swmm_views/12_vw_swmm_raingages.sql @@ -6,7 +6,7 @@ SELECT '0:15'::varchar as Interval, '1.0'::varchar as SCF, 'TIMESERIES default_tww_raingage_timeserie'::varchar as Source, - st_centroid(perimeter_geometry)::geometry(Point, %(SRID)s) as geom, + st_centroid(perimeter_geometry)::geometry(Point, {SRID}) as geom, state, CASE WHEN _function_hierarchic in (5062, 5064, 5066, 5068, 5069, 5070, 5071, 5072, 5074) THEN 'primary' diff --git a/datamodel/app/view/catchment_area/vw_catchment_area_connections.sql b/datamodel/app/view/catchment_area/vw_catchment_area_connections.sql index 6cac10fbb..645a37490 100644 --- a/datamodel/app/view/catchment_area/vw_catchment_area_connections.sql +++ b/datamodel/app/view/catchment_area/vw_catchment_area_connections.sql @@ -5,9 +5,9 @@ SELECT ca.obj_id, ST_Force2D(ST_MakeLine(ST_Centroid(ST_CurveToLine(perimeter_geometry)), -wn_rw_current.situation3d_geometry))::geometry( LineString, %(SRID)s ) AS connection_rw_current_geometry, +wn_rw_current.situation3d_geometry))::geometry( LineString, {SRID} ) AS connection_rw_current_geometry, ST_Force2D(ST_MakeLine(ST_Centroid(ST_CurveToLine(perimeter_geometry)), -wn_ww_current.situation3d_geometry))::geometry( LineString, %(SRID)s ) AS connection_ww_current_geometry +wn_ww_current.situation3d_geometry))::geometry( LineString, {SRID} ) AS connection_ww_current_geometry FROM tww_od.catchment_area ca LEFT JOIN tww_od.wastewater_node wn_rw_current diff --git a/datamodel/app/view/create_views.py b/datamodel/app/view/create_views.py index a501d19c3..0eec98217 100755 --- a/datamodel/app/view/create_views.py +++ b/datamodel/app/view/create_views.py @@ -2,7 +2,7 @@ import argparse -import psycopg2 +import psycopg from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance from vw_tww_reach import vw_tww_reach from vw_tww_wastewater_structure import vw_tww_wastewater_structure @@ -13,9 +13,9 @@ def run_sql(file_path: str, pg_service: str, variables: dict = {}): sql = open(file_path).read() - conn = psycopg2.connect(f"service={pg_service}") + conn = psycopg.connect(f"service={pg_service}") cursor = conn.cursor() - cursor.execute(sql, variables) + cursor.execute(psycopg.sql.SQL(sql).format(**variables)) conn.commit() conn.close() diff --git a/datamodel/app/view/vw_change_points.sql b/datamodel/app/view/vw_change_points.sql index eb0faef79..81b4048e1 100644 --- a/datamodel/app/view/vw_change_points.sql +++ b/datamodel/app/view/vw_change_points.sql @@ -1,7 +1,7 @@ CREATE VIEW tww_app.vw_change_points AS SELECT rp_to.obj_id, - rp_to.situation3d_geometry::geometry(POINTZ, %(SRID)s) AS geom, + rp_to.situation3d_geometry::geometry(POINTZ, {SRID}) AS geom, re.material <> re_next.material AS change_in_material, re.clear_height <> re_next.clear_height AS change_in_clear_height, (rp_from.level - rp_to.level) / re.length_effective - (rp_next_from.level - rp_next_to.level) / re_next.length_effective AS change_in_slope diff --git a/datamodel/app/view/vw_oo_overflow.yaml b/datamodel/app/view/vw_oo_overflow.yaml index f3af568ad..3a90a9e60 100755 --- a/datamodel/app/view/vw_oo_overflow.yaml +++ b/datamodel/app/view/vw_oo_overflow.yaml @@ -4,7 +4,7 @@ view_name: vw_tww_overflow view_schema: tww_app additional_columns: - geometry: n1.situation3d_geometry::geometry('PointZ',%(SRID)s) + geometry: n1.situation3d_geometry::geometry('PointZ',{SRID}) additional_joins: > LEFT JOIN tww_od.wastewater_node n1 ON overflow.fk_wastewater_node::text = n1.obj_id::text diff --git a/datamodel/app/view/vw_tww_import.sql b/datamodel/app/view/vw_tww_import.sql index 63a3ef11d..d61779208 100644 --- a/datamodel/app/view/vw_tww_import.sql +++ b/datamodel/app/view/vw_tww_import.sql @@ -3,7 +3,7 @@ CREATE OR REPLACE VIEW tww_app.import_vw_manhole AS SELECT DISTINCT ON (ws.obj_id) ws.obj_id, ws.identifier, - ST_Force3D(ws.situation3d_geometry)::geometry(POINTZ, %(SRID)s) AS situation_geometry, + ST_Force3D(ws.situation3d_geometry)::geometry(POINTZ, {SRID}) AS situation_geometry, ws.co_shape, ws.co_diameter, ws.co_material, @@ -301,11 +301,11 @@ BEGIN -- catch EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'EXCEPTION: %%%%', SQLERRM; + RAISE NOTICE 'EXCEPTION: %%', SQLERRM; RETURN NEW; END; $BODY$ LANGUAGE plpgsql; -$TRIGGER$, %(SRID)s); +$TRIGGER$, {SRID}); END $DO$; @@ -318,7 +318,7 @@ CREATE TRIGGER after_update_try_structure_update WHEN ( ( NEW.structure_okay IS NOT TRUE ) AND NOT( OLD.inlet_okay IS NOT TRUE AND NEW.inlet_okay IS TRUE ) AND NOT( OLD.outlet_okay IS NOT TRUE AND NEW.outlet_okay IS TRUE ) ) - EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update(%(SRID)s); + EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update({SRID}); DROP TRIGGER IF EXISTS after_insert_try_structure_update ON tww_od.import_manhole_quarantine; @@ -326,7 +326,7 @@ CREATE TRIGGER after_insert_try_structure_update AFTER INSERT ON tww_od.import_manhole_quarantine FOR EACH ROW - EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update(%(SRID)s); + EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update({SRID}); -- Some information: -- 1. new lets 0 - old lets 0 -> do nothing @@ -370,14 +370,14 @@ BEGIN -- handle inlets IF ( new_lets > 1 AND old_lets > 0 ) OR old_lets > 1 THEN -- request for update because new lets are bigger 1 (and old lets not 0 ) or old lets are bigger 1 - RAISE NOTICE 'Impossible to assign %%s - manual edit needed.', let_kind; + RAISE NOTICE 'Impossible to assign %s - manual edit needed.', let_kind; ELSE IF new_lets = 0 AND old_lets > 0 THEN -- request for delete because no new lets but old lets - RAISE NOTICE 'No new %%s but old ones - manual delete needed.', let_kind; + RAISE NOTICE 'No new %s but old ones - manual delete needed.', let_kind; ELSIF new_lets > 0 AND old_lets = 0 THEN -- request for create because no old lets but new lets - RAISE NOTICE 'No old %%s but new ones - manual create needed.', let_kind; + RAISE NOTICE 'No old %s but new ones - manual create needed.', let_kind; ELSE IF new_lets = 1 AND old_lets = 1 THEN IF let_kind='inlet' THEN @@ -424,10 +424,10 @@ BEGIN WHERE ws.obj_id = NEW.obj_id ); END IF; - RAISE NOTICE '%%s updated', let_kind; + RAISE NOTICE '%s updated', let_kind; ELSE -- do nothing - RAISE NOTICE 'No %%s - nothing to do', let_kind; + RAISE NOTICE 'No %s - nothing to do', let_kind; END IF; IF let_kind='inlet' THEN @@ -448,7 +448,7 @@ BEGIN -- catch EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'EXCEPTION: %%', SQLERRM; + RAISE NOTICE 'EXCEPTION: %', SQLERRM; RETURN NEW; END; $BODY$ LANGUAGE plpgsql; diff --git a/datamodel/app/view/vw_tww_reach.py b/datamodel/app/view/vw_tww_reach.py index 4c2e0110c..9fad75385 100644 --- a/datamodel/app/view/vw_tww_reach.py +++ b/datamodel/app/view/vw_tww_reach.py @@ -5,7 +5,7 @@ import argparse import os -import psycopg2 +import psycopg from pirogue.utils import insert_command, select_columns, table_parts, update_command from yaml import safe_load @@ -21,7 +21,7 @@ def vw_tww_reach(pg_service: str = None, extra_definition: dict = None): assert pg_service extra_definition = extra_definition or {} - conn = psycopg2.connect(f"service={pg_service}") + conn = psycopg.connect(f"service={pg_service}") cursor = conn.cursor() view_sql = """ diff --git a/datamodel/app/view/vw_tww_wastewater_structure.py b/datamodel/app/view/vw_tww_wastewater_structure.py index d7d00c381..9a8137a52 100644 --- a/datamodel/app/view/vw_tww_wastewater_structure.py +++ b/datamodel/app/view/vw_tww_wastewater_structure.py @@ -5,7 +5,7 @@ import argparse import os -import psycopg2 +import psycopg from pirogue.utils import insert_command, select_columns, table_parts, update_command from yaml import safe_load @@ -22,9 +22,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit assert pg_service extra_definition = extra_definition or {} - variables = {"SRID": int(srid)} - - conn = psycopg2.connect(f"service={pg_service}") + conn = psycopg.connect(f"service={pg_service}") cursor = conn.cursor() view_sql = """ @@ -56,7 +54,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit , main_co_sp.renovation_demand AS co_renovation_demand , {main_co_cols} - , ST_Force2D(COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry))::geometry(Point, %(SRID)s) AS situation3d_geometry + , ST_Force2D(COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry))::geometry(Point, {srid}) AS situation3d_geometry , {ma_columns} @@ -94,6 +92,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover'); ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node'); """.format( + srid=srid, extra_cols="\n ".join( [ select_columns( @@ -221,7 +220,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit ), ) - cursor.execute(view_sql, variables) + cursor.execute(view_sql) trigger_insert_sql = """ CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_wastewater_structure_INSERT() @@ -438,7 +437,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit SET situation3d_geometry = ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )), - ST_Z(WN.situation3d_geometry)), %(SRID)s ) + ST_Z(WN.situation3d_geometry)), {srid} ) WHERE obj_id IN ( SELECT obj_id FROM tww_od.wastewater_networkelement @@ -450,7 +449,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit SET situation3d_geometry = ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )), - ST_Z(CO.situation3d_geometry)), %(SRID)s ) + ST_Z(CO.situation3d_geometry)), {srid} ) WHERE obj_id IN ( SELECT obj_id FROM tww_od.structure_part @@ -466,7 +465,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )), - ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s ) + ST_Z(ST_PointN(RE.progression3d_geometry, 1))), {srid} ) ) ) WHERE fk_reach_point_from IN ( @@ -483,7 +482,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit ST_SetSRID( ST_MakePoint( ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )), ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )), - ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s ) + ST_Z(ST_PointN(RE.progression3d_geometry, 1))), {srid} ) ) ) WHERE fk_reach_point_to IN ( @@ -505,6 +504,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit CREATE TRIGGER vw_tww_wastewater_structure_UPDATE INSTEAD OF UPDATE ON tww_app.vw_tww_wastewater_structure FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_wastewater_structure_UPDATE(); """.format( + srid=srid, update_co=update_command( pg_cur=cursor, table_schema="tww_od", @@ -605,7 +605,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit ), ) - cursor.execute(update_trigger_sql, variables) + cursor.execute(update_trigger_sql) trigger_delete_sql = """ CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_wastewater_structure_DELETE() @@ -622,7 +622,8 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit CREATE TRIGGER vw_tww_wastewater_structure_DELETE INSTEAD OF DELETE ON tww_app.vw_tww_wastewater_structure FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_wastewater_structure_DELETE(); """ - cursor.execute(trigger_delete_sql, variables) + + cursor.execute(trigger_delete_sql) extras = """ ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure'); diff --git a/datamodel/requirements.txt b/datamodel/requirements.txt index d84e3477c..6ab27d9e7 100644 --- a/datamodel/requirements.txt +++ b/datamodel/requirements.txt @@ -1,3 +1,3 @@ -pirogue>=1.4.2 -psycopg2 +pirogue==2.0.1b3 +psycopg>=3.1.18 pyyaml diff --git a/datamodel/test/test_geometry.py b/datamodel/test/test_geometry.py index d00cb1bc5..7e362bbb3 100644 --- a/datamodel/test/test_geometry.py +++ b/datamodel/test/test_geometry.py @@ -2,8 +2,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -19,7 +18,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_vw_tww_reach_geometry_insert(self): # 1. insert geometry with Z and no rp_from_level and no rp_to_level diff --git a/datamodel/test/test_import.py b/datamodel/test/test_import.py index 0486383c4..ab6947fb1 100644 --- a/datamodel/test/test_import.py +++ b/datamodel/test/test_import.py @@ -2,9 +2,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras -import psycopg2.sql +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -17,7 +15,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") # - level calculation failing because only no reference level # -> not updated structure with calculated values @@ -207,14 +205,14 @@ def test_update_with_outlet(self): # it should be in the live table tww_od.reach and tww_od.reach_point cur = self.cursor() cur.execute( - psycopg2.sql.SQL( + psycopg.sql.SQL( "SELECT re.material, re.clear_height, rp.level, ws.co_level\ FROM {schema}.reach re\ LEFT JOIN {schema}.reach_point rp ON rp.obj_id = re.fk_reach_point_from\ LEFT JOIN {schema}.wastewater_networkelement wn ON wn.obj_id = rp.fk_wastewater_networkelement\ LEFT JOIN {schema}.vw_tww_wastewater_structure ws ON ws.obj_id = wn.fk_wastewater_structure\ WHERE ws.obj_id = %(obj_id)s" - ).format(schema=psycopg2.sql.Identifier("tww_od")), + ).format(schema=psycopg.sql.Identifier("tww_od")), {"obj_id": obj_id}, ) row = cur.fetchone() @@ -227,11 +225,11 @@ def test_update_with_outlet(self): row = self.select("file", obj_id, "tww_od") cur = self.cursor() cur.execute( - psycopg2.sql.SQL( + psycopg.sql.SQL( "SELECT *\ FROM {schema}.file\ WHERE object = %(obj_id)s" - ).format(schema=psycopg2.sql.Identifier("tww_od")), + ).format(schema=psycopg.sql.Identifier("tww_od")), {"obj_id": obj_id}, ) row = cur.fetchone() @@ -262,7 +260,7 @@ def test_update_with_wrong_material(self): # it should be in the live table tww_od.reach and tww_od.reach_point cur = self.cursor() cur.execute( - psycopg2.sql.SQL( + psycopg.sql.SQL( "SELECT re.material, re.clear_height, rp.level, ws.co_level\ FROM tww_od.reach re\ LEFT JOIN tww_od.reach_point rp ON rp.obj_id = re.fk_reach_point_from\ @@ -339,14 +337,14 @@ def test_update_with_unexpected_inlet(self): # it should be in the live table tww_od.reach and tww_od.reach_point cur = self.cursor() cur.execute( - psycopg2.sql.SQL( + psycopg.sql.SQL( "SELECT re.material, re.clear_height, rp.level, ws.co_level\ FROM {schema}.reach re\ LEFT JOIN {schema}.reach_point rp ON rp.obj_id = re.fk_reach_point_from\ LEFT JOIN {schema}.wastewater_networkelement wn ON wn.obj_id = rp.fk_wastewater_networkelement\ LEFT JOIN {schema}.vw_tww_wastewater_structure ws ON ws.obj_id = wn.fk_wastewater_structure\ WHERE ws.obj_id = %(obj_id)s" - ).format(schema=psycopg2.sql.Identifier("tww_od")), + ).format(schema=psycopg.sql.Identifier("tww_od")), {"obj_id": obj_id}, ) row = cur.fetchone() @@ -442,14 +440,14 @@ def test_update_with_multiple_inlets(self): # it should be in the live table tww_od.reach and tww_od.reach_point cur = self.cursor() cur.execute( - psycopg2.sql.SQL( + psycopg.sql.SQL( "SELECT re.material, re.clear_height, rp.level, ws.co_level\ FROM {schema}.reach re\ LEFT JOIN {schema}.reach_point rp ON rp.obj_id = re.fk_reach_point_from\ LEFT JOIN {schema}.wastewater_networkelement wn ON wn.obj_id = rp.fk_wastewater_networkelement\ LEFT JOIN {schema}.vw_tww_wastewater_structure ws ON ws.obj_id = wn.fk_wastewater_structure\ WHERE ws.obj_id = %(obj_id)s" - ).format(schema=psycopg2.sql.Identifier("tww_od")), + ).format(schema=psycopg.sql.Identifier("tww_od")), {"obj_id": obj_id}, ) row = cur.fetchone() @@ -477,8 +475,8 @@ def test_general(self): # it should be in the live table tww_od.reach and tww_od.reach_point cur = self.cursor() cur.execute( - psycopg2.sql.SQL("SELECT * FROM {schema}.wastewater_structure LIMIT 1").format( - schema=psycopg2.sql.Identifier("tww_od") + psycopg.sql.SQL("SELECT * FROM {schema}.wastewater_structure LIMIT 1").format( + schema=psycopg.sql.Identifier("tww_od") ) ) row = cur.fetchone() diff --git a/datamodel/test/test_label.py b/datamodel/test/test_label.py index 529c98423..2e58bfb9b 100644 --- a/datamodel/test/test_label.py +++ b/datamodel/test/test_label.py @@ -1,8 +1,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -15,7 +14,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def insert_manholes(self, manholes): for manhole in manholes: diff --git a/datamodel/test/test_network.py b/datamodel/test/test_network.py index d89c7f82a..374b35dcb 100644 --- a/datamodel/test/test_network.py +++ b/datamodel/test/test_network.py @@ -1,8 +1,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -28,7 +27,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def make_reach(self, identifier, x1, y1, x2, y2): """ @@ -89,7 +88,7 @@ def downstream_nodes_depths(self, node_id): SELECT obj_id, depth FROM downstream; """ - cur = self.cursor() + cur = self.cursor(row_factory=psycopg.rows.dict_row) cur.execute(query, (node_id,)) return {row["obj_id"]: row["depth"] for row in cur.fetchall()} @@ -116,7 +115,7 @@ def upstream_nodes_depths(self, node_id): SELECT obj_id, depth FROM upstream """ - cur = self.cursor() + cur = self.cursor(row_factory=psycopg.rows.dict_row) cur.execute(query, (node_id,)) return {row["obj_id"]: row["depth"] for row in cur.fetchall()} diff --git a/datamodel/test/test_on_delete.py b/datamodel/test/test_on_delete.py index 41b7525ea..0365512e3 100644 --- a/datamodel/test/test_on_delete.py +++ b/datamodel/test/test_on_delete.py @@ -2,8 +2,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -16,7 +15,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_delete_wastewater_structure(self): # Create a new cover(structure part) with manhole(wastewater structure) diff --git a/datamodel/test/test_removed_fields.py b/datamodel/test/test_removed_fields.py index fe505b645..9850fc85e 100644 --- a/datamodel/test/test_removed_fields.py +++ b/datamodel/test/test_removed_fields.py @@ -1,8 +1,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -15,10 +14,10 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_dataowner(self): - cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = self.conn.cursor(row_factory=psycopg.rows.dict_row) cur.execute("SELECT * FROM tww_od.wastewater_structure LIMIT 1") colnames = [desc[0] for desc in cur.description] diff --git a/datamodel/test/test_schemas.py b/datamodel/test/test_schemas.py index 9bb24db33..43b460853 100644 --- a/datamodel/test/test_schemas.py +++ b/datamodel/test/test_schemas.py @@ -1,8 +1,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -18,7 +17,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_list_schemas(self): schemas = ", ".join([f"'{schema}'" for schema in TWW_SCHEMAS + PG_SCHEMAS]) diff --git a/datamodel/test/test_swmm.py b/datamodel/test/test_swmm.py index e3755442a..cb4f9b7d9 100644 --- a/datamodel/test/test_swmm.py +++ b/datamodel/test/test_swmm.py @@ -1,8 +1,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -19,7 +18,7 @@ def tearDown(cls): @classmethod def setUp(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_count_vw_aquifers(self): self.assert_count("swmm_vw_aquifers", "tww_app", 0) diff --git a/datamodel/test/test_symbology.py b/datamodel/test/test_symbology.py index e3691b364..81d6fc745 100644 --- a/datamodel/test/test_symbology.py +++ b/datamodel/test/test_symbology.py @@ -1,8 +1,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -15,7 +14,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_create_drop_triggers(self): self.execute("tww_sys.disable_symbology_triggers()") diff --git a/datamodel/test/test_triggers.py b/datamodel/test/test_triggers.py index 4677837dc..637429580 100644 --- a/datamodel/test/test_triggers.py +++ b/datamodel/test/test_triggers.py @@ -2,8 +2,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -16,7 +15,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_last_modified(self): row = { diff --git a/datamodel/test/test_views.py b/datamodel/test/test_views.py index d2acaaba5..389c00b4f 100644 --- a/datamodel/test/test_views.py +++ b/datamodel/test/test_views.py @@ -3,8 +3,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DEFAULT_PG_SERVICE, DbTestBase @@ -17,7 +16,7 @@ def tearDownClass(cls): @classmethod def setUpClass(cls): pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE - cls.conn = psycopg2.connect(f"service={pgservice}") + cls.conn = psycopg.connect(f"service={pgservice}") def test_vw_reach(self): row = { @@ -93,7 +92,7 @@ def test_vw_tww_wastewater_structure(self): self.update_check("vw_tww_wastewater_structure", row, obj_id) - cur = self.cursor() + cur = self.cursor(row_factory=psycopg.rows.dict_row) cur.execute( "SELECT * FROM tww_od.wastewater_networkelement NE LEFT JOIN tww_od.wastewater_node NO ON NO.obj_id = NE.obj_id WHERE fk_wastewater_structure='{obj_id}' ".format( diff --git a/datamodel/test/utils.py b/datamodel/test/utils.py index d2765c5da..9ed6c5889 100644 --- a/datamodel/test/utils.py +++ b/datamodel/test/utils.py @@ -1,5 +1,4 @@ -import psycopg2 -import psycopg2.extras +import psycopg DEFAULT_PG_SERVICE = "pg_tww" @@ -7,7 +6,7 @@ class DbTestBase: @classmethod def assert_count(cls, table, schema, expected): - cur = cls.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = cls.conn.cursor() cur.execute(f"SELECT COUNT(*) FROM {schema}.{table}") count = cur.fetchone()[0] assert ( @@ -24,26 +23,26 @@ def geom_as_text(cls, wkb: str) -> str: @classmethod def select(cls, table, obj_id, schema="tww_app"): - cur = cls.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = cls.conn.cursor(row_factory=psycopg.rows.dict_row) cur.execute(f"SELECT * FROM {schema}.{table} WHERE obj_id=%(obj_id)s", {"obj_id": obj_id}) return cur.fetchone() @classmethod def execute(cls, sql: str, params=[]): - cur = cls.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = cls.conn.cursor() if not sql.startswith("SELECT"): sql = f"SELECT {sql}" cur.execute(sql, params) return cur.fetchone()[0] def check_empty(self, sql: str): - cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = self.conn.cursor() cur.execute(sql) self.assertIsNone(cur.fetchone()) @classmethod - def cursor(cls): - return cls.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + def cursor(cls, **kwargs): + return cls.conn.cursor(**kwargs) @classmethod def insert(cls, table, row, schema="tww_app"): diff --git a/datamodel/update/post-all.py b/datamodel/update/post-all.py index 861c35684..458f59bee 100644 --- a/datamodel/update/post-all.py +++ b/datamodel/update/post-all.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import psycopg2 +import psycopg from pum.core.deltapy import DeltaPy from view.create_views import create_views @@ -25,7 +25,7 @@ def run(self): ) # refresh network views - conn = psycopg2.connect(f"service={self.pg_service}") + conn = psycopg.connect(f"service={self.pg_service}") cursor = conn.cursor() cursor.execute("SELECT tww_app.network_refresh_network_simple();") conn.commit() diff --git a/docs/en/installation-guide/requirements.rst b/docs/en/installation-guide/requirements.rst index 6bf240b47..95abf39ca 100644 --- a/docs/en/installation-guide/requirements.rst +++ b/docs/en/installation-guide/requirements.rst @@ -5,6 +5,6 @@ Requirements The following Python packages (installed using `pip `_) are required: -* `Psycopg2 `_ as database adapter +* `Psycopg `_ as database adapter * `Pirogue `_ for model generation and upgrades * `PUM `_ for model upgrades diff --git a/plugin/teksi_wastewater/interlis/interlis_importer_exporter.py b/plugin/teksi_wastewater/interlis/interlis_importer_exporter.py index ea618ca00..c14cd2439 100644 --- a/plugin/teksi_wastewater/interlis/interlis_importer_exporter.py +++ b/plugin/teksi_wastewater/interlis/interlis_importer_exporter.py @@ -2,7 +2,7 @@ import os import tempfile -import psycopg2 +import psycopg from PyQt5.QtCore import Qt from PyQt5.QtWidgets import QApplication @@ -26,7 +26,7 @@ from .utils.various import ( CmdException, LoggingHandlerContext, - get_pgconf_as_psycopg2_dsn, + get_pgconf_as_psycopg_dsn, logger, make_log_path, ) @@ -233,7 +233,7 @@ def _import_from_intermediate_schema(self, import_model): return interlisImporterToIntermediateSchema.session_tww def _import_update_main_cover_and_refresh_mat_views(self): - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection = psycopg.connect(get_pgconf_as_psycopg_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() @@ -386,7 +386,7 @@ def _export_xtf_files(self, file_name_base, export_models): def _clear_ili_schema(self, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection = psycopg.connect(get_pgconf_as_psycopg_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() diff --git a/plugin/teksi_wastewater/interlis/utils/various.py b/plugin/teksi_wastewater/interlis/utils/various.py index 2fb0abffd..b9ce48f41 100644 --- a/plugin/teksi_wastewater/interlis/utils/various.py +++ b/plugin/teksi_wastewater/interlis/utils/various.py @@ -236,8 +236,8 @@ def get_pgconf(): return collections.defaultdict(str, pgconf) -def get_pgconf_as_psycopg2_dsn() -> List[str]: - """Returns the pgconf as a psycopg2 connection string""" +def get_pgconf_as_psycopg_dsn() -> List[str]: + """Returns the pgconf as a psycopg connection string""" pgconf = get_pgconf() parts = [] diff --git a/plugin/teksi_wastewater/processing_provider/TwwSwmm.py b/plugin/teksi_wastewater/processing_provider/TwwSwmm.py index 2a344125a..7faa4a8e7 100644 --- a/plugin/teksi_wastewater/processing_provider/TwwSwmm.py +++ b/plugin/teksi_wastewater/processing_provider/TwwSwmm.py @@ -21,7 +21,7 @@ import subprocess from datetime import datetime, timedelta -import psycopg2 +import psycopg MEASURING_POINT_KIND = "Diverse kind of SWMM simulation parameters" MEASURING_DEVICE_REMARK = "SWMM Simulation" @@ -134,7 +134,7 @@ def __init__(self, title, service, state, inpfile, inptemplate, rptfile, binfile def __enter__(self): if self.service is not None: - self.con = psycopg2.connect(service=self.service) + self.con = psycopg.connect(service=self.service) return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -207,7 +207,7 @@ def get_swmm_table(self, table_name, state, selected_structures, hierarchy): ) try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while executing: {sql}") return None, None self.feedback_push("info", f"Process vw_{table_name}") @@ -872,9 +872,9 @@ def populate_attribute(self, data, table_name, attribute_name, swmm_attribute): ) try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", str(psycopg2.ProgrammingError)) + self.feedback_push("error", str(psycopg.ProgrammingError)) return None, None res = cur.fetchone() if res is None: @@ -937,9 +937,9 @@ def create_measuring_point_node(self, node_obj_id, sim_description): ) try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", str(psycopg2.ProgrammingError)) + self.feedback_push("error", str(psycopg.ProgrammingError)) return None, None res = cur.fetchone() if res is None: @@ -1001,9 +1001,9 @@ def create_measuring_point_link(self, reach_obj_id, sim_description): ) try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", str(psycopg2.ProgrammingError)) + self.feedback_push("error", str(psycopg.ProgrammingError)) return None res = cur.fetchone() mp_obj_id = res[0] @@ -1052,9 +1052,9 @@ def create_measuring_device(self, mp_obj_id): ) try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", str(psycopg2.ProgrammingError)) + self.feedback_push("error", str(psycopg.ProgrammingError)) return None, None res = cur.fetchone() if res is None: @@ -1115,9 +1115,9 @@ def create_measurement_series(self, mp_obj_id, parameter_name, parameter_dimensi try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", str(psycopg2.ProgrammingError)) + self.feedback_push("error", str(psycopg.ProgrammingError)) return None ms_obj_id = cur.fetchone()[0] self.con.commit() @@ -1158,9 +1158,9 @@ def create_measurement_result( ) try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", (str(psycopg2.ProgrammingError))) + self.feedback_push("error", (str(psycopg.ProgrammingError))) return None res = cur.fetchone() @@ -1184,9 +1184,9 @@ def create_measurement_result( try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", (str(psycopg2.ProgrammingError))) + self.feedback_push("error", (str(psycopg.ProgrammingError))) return None mr_obj_id = cur.fetchone()[0] self.con.commit() @@ -1203,9 +1203,9 @@ def create_measurement_result( ) try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", (str(psycopg2.ProgrammingError))) + self.feedback_push("error", (str(psycopg.ProgrammingError))) return None mr_obj_id = cur.fetchone()[0] self.con.commit() @@ -1225,9 +1225,9 @@ def disable_reach_trigger(self): """ try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", (str(psycopg2.ProgrammingError))) + self.feedback_push("error", (str(psycopg.ProgrammingError))) return None self.con.commit() del cur @@ -1246,9 +1246,9 @@ def enable_reach_trigger(self): """ try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", (str(psycopg2.ProgrammingError))) + self.feedback_push("error", (str(psycopg.ProgrammingError))) return None self.con.commit() del cur @@ -1270,9 +1270,9 @@ def set_reach_default_friction(self): """ try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", (str(psycopg2.ProgrammingError))) + self.feedback_push("error", (str(psycopg.ProgrammingError))) return None self.con.commit() del cur @@ -1294,9 +1294,9 @@ def overwrite_reach_default_friction(self): """ try: cur.execute(sql) - except psycopg2.ProgrammingError: + except psycopg.ProgrammingError: self.feedback_push("error", f"Error while excecuting: {sql}") - self.feedback_push("error", (str(psycopg2.ProgrammingError))) + self.feedback_push("error", (str(psycopg.ProgrammingError))) return None self.con.commit() del cur