upgrade to psycopg3

.github/workflows/plugin-test.yml

@@ -40,6 +40,9 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Setup env
+        run: cp .env.example .env
+
       - name: Package PyPI Packages
         run: sudo ./plugin/scripts/package-pip-packages.sh
 

datamodel/.docker/Dockerfile

@@ -27,7 +27,6 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 RUN pip install -r datamodel/requirements.txt
 RUN if [ "${RUN_TEST}" = "True" ]; then pip install -r datamodel/requirements-test.txt; fi
 
-
 # Configure the postgres connections
 RUN printf '[postgres]\ndbname=postgres\nuser=postgres\n' >> /etc/postgresql-common/pg_service.conf
 RUN printf '[pg_tww]\ndbname=tww\nuser=postgres\n' >> /etc/postgresql-common/pg_service.conf
@@ -41,7 +40,7 @@ RUN if [ "${AUTO_INIT}" = "True" ]; then ln -s /src/datamodel/.docker/init_db.sh
 
 # Some defaults
 ENV POSTGRES_PASSWORD=postgres
-# otherwise psycopg2 cannot connect
+# otherwise psycopg cannot connect
 ENV PGSERVICEFILE=/etc/postgresql-common/pg_service.conf
 
 ENV PGSERVICE=pg_tww

datamodel/app/swmm_views/04_vw_swmm_conduits.sql

@@ -113,7 +113,7 @@ SELECT
 	END
 	) as description,
 	cfhy.value_en as tag,
-	ST_CurveToLine(st_force3d(progression3d_geometry))::geometry(LineStringZ, %(SRID)s) as geom,
+	ST_CurveToLine(st_force3d(progression3d_geometry))::geometry(LineStringZ, {SRID}) as geom,
 	CASE
 		WHEN status IN (7959, 6529, 6526) THEN 'planned'
 		ELSE 'current'

datamodel/app/swmm_views/09_vw_swmm_subcatchments.sql

@@ -69,7 +69,7 @@ SELECT
 		THEN NULL
 	END as description,
   ca.obj_id as tag,
-  ST_CurveToLine(perimeter_geometry)::geometry(Polygon, %(SRID)s) as geom,
+  ST_CurveToLine(perimeter_geometry)::geometry(Polygon, {SRID}) as geom,
   CASE
     WHEN state = 'rw_current' OR state = 'ww_current' THEN 'current'
     WHEN state = 'rw_planned' OR state = 'ww_planned' THEN 'planned'

datamodel/app/swmm_views/12_vw_swmm_raingages.sql

@@ -6,7 +6,7 @@ SELECT
   '0:15'::varchar as Interval,
   '1.0'::varchar as SCF,
   'TIMESERIES default_tww_raingage_timeserie'::varchar as Source,
-  st_centroid(perimeter_geometry)::geometry(Point, %(SRID)s) as geom,
+  st_centroid(perimeter_geometry)::geometry(Point, {SRID}) as geom,
   state,
   CASE
 		WHEN _function_hierarchic in (5062, 5064, 5066, 5068, 5069, 5070, 5071, 5072, 5074) THEN 'primary'

datamodel/app/view/catchment_area/vw_catchment_area_connections.sql

@@ -5,9 +5,9 @@ SELECT
 
 ca.obj_id,
 ST_Force2D(ST_MakeLine(ST_Centroid(ST_CurveToLine(perimeter_geometry)),
-wn_rw_current.situation3d_geometry))::geometry( LineString, %(SRID)s ) AS connection_rw_current_geometry,
+wn_rw_current.situation3d_geometry))::geometry( LineString, {SRID} ) AS connection_rw_current_geometry,
 ST_Force2D(ST_MakeLine(ST_Centroid(ST_CurveToLine(perimeter_geometry)),
-wn_ww_current.situation3d_geometry))::geometry( LineString, %(SRID)s ) AS connection_ww_current_geometry
+wn_ww_current.situation3d_geometry))::geometry( LineString, {SRID} ) AS connection_ww_current_geometry
 
 FROM tww_od.catchment_area ca
 LEFT JOIN tww_od.wastewater_node wn_rw_current

datamodel/app/view/create_views.py

@@ -2,7 +2,7 @@
 
 import argparse
 
-import psycopg2
+import psycopg
 from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance
 from vw_tww_reach import vw_tww_reach
 from vw_tww_wastewater_structure import vw_tww_wastewater_structure
@@ -13,9 +13,9 @@
 
 def run_sql(file_path: str, pg_service: str, variables: dict = {}):
     sql = open(file_path).read()
-    conn = psycopg2.connect(f"service={pg_service}")
+    conn = psycopg.connect(f"service={pg_service}")
     cursor = conn.cursor()
-    cursor.execute(sql, variables)
+    cursor.execute(psycopg.sql.SQL(sql).format(**variables))
     conn.commit()
     conn.close()
 

datamodel/app/view/vw_change_points.sql

@@ -1,7 +1,7 @@
 CREATE VIEW tww_app.vw_change_points AS
 SELECT
   rp_to.obj_id,
-  rp_to.situation3d_geometry::geometry(POINTZ, %(SRID)s) AS geom,
+  rp_to.situation3d_geometry::geometry(POINTZ, {SRID}) AS geom,
   re.material <> re_next.material AS change_in_material,
   re.clear_height <> re_next.clear_height AS change_in_clear_height,
   (rp_from.level - rp_to.level) / re.length_effective - (rp_next_from.level - rp_next_to.level) / re_next.length_effective AS change_in_slope

datamodel/app/view/vw_oo_overflow.yaml

@@ -4,7 +4,7 @@ view_name: vw_tww_overflow
 view_schema: tww_app
 
 additional_columns:
-  geometry: n1.situation3d_geometry::geometry('PointZ',%(SRID)s)
+  geometry: n1.situation3d_geometry::geometry('PointZ',{SRID})
 additional_joins: >
   LEFT JOIN tww_od.wastewater_node n1 ON overflow.fk_wastewater_node::text = n1.obj_id::text
 

datamodel/app/view/vw_tww_import.sql

@@ -3,7 +3,7 @@
 CREATE OR REPLACE VIEW tww_app.import_vw_manhole AS
  SELECT DISTINCT ON (ws.obj_id) ws.obj_id,
     ws.identifier,
-    ST_Force3D(ws.situation3d_geometry)::geometry(POINTZ, %(SRID)s) AS situation_geometry,
+    ST_Force3D(ws.situation3d_geometry)::geometry(POINTZ, {SRID}) AS situation_geometry,
     ws.co_shape,
     ws.co_diameter,
     ws.co_material,
@@ -301,11 +301,11 @@ BEGIN
 
   -- catch
   EXCEPTION WHEN OTHERS THEN
-    RAISE NOTICE 'EXCEPTION: %%%%', SQLERRM;
+    RAISE NOTICE 'EXCEPTION: %%', SQLERRM;
     RETURN NEW;
 END; $BODY$
 LANGUAGE plpgsql;
-$TRIGGER$, %(SRID)s);
+$TRIGGER$, {SRID});
 END
 $DO$;
 
@@ -318,15 +318,15 @@ CREATE TRIGGER after_update_try_structure_update
   WHEN ( ( NEW.structure_okay IS NOT TRUE )
   AND NOT( OLD.inlet_okay IS NOT TRUE AND NEW.inlet_okay IS TRUE )
   AND NOT( OLD.outlet_okay IS NOT TRUE AND NEW.outlet_okay IS TRUE ) )
-  EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update(%(SRID)s);
+  EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update({SRID});
 
 DROP TRIGGER IF EXISTS after_insert_try_structure_update ON tww_od.import_manhole_quarantine;
 
 CREATE TRIGGER after_insert_try_structure_update
   AFTER INSERT
   ON tww_od.import_manhole_quarantine
   FOR EACH ROW
-  EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update(%(SRID)s);
+  EXECUTE PROCEDURE tww_app.import_manhole_quarantine_try_structure_update({SRID});
 
 -- Some information:
 -- 1. new lets 0 - old lets 0 -> do nothing
@@ -370,14 +370,14 @@ BEGIN
   -- handle inlets
   IF ( new_lets > 1 AND old_lets > 0 ) OR old_lets > 1 THEN
     -- request for update because new lets are bigger 1 (and old lets not 0 ) or old lets are bigger 1
-    RAISE NOTICE 'Impossible to assign %%s - manual edit needed.', let_kind;
+    RAISE NOTICE 'Impossible to assign %s - manual edit needed.', let_kind;
   ELSE
     IF new_lets = 0 AND old_lets > 0 THEN
       -- request for delete because no new lets but old lets
-      RAISE NOTICE 'No new %%s but old ones - manual delete needed.', let_kind;
+      RAISE NOTICE 'No new %s but old ones - manual delete needed.', let_kind;
     ELSIF new_lets > 0 AND old_lets = 0 THEN
       -- request for create because no old lets but new lets
-      RAISE NOTICE 'No old %%s but new ones - manual create needed.', let_kind;
+      RAISE NOTICE 'No old %s but new ones - manual create needed.', let_kind;
     ELSE
       IF new_lets = 1 AND old_lets = 1 THEN
         IF let_kind='inlet' THEN
@@ -424,10 +424,10 @@ BEGIN
             WHERE ws.obj_id = NEW.obj_id );
         END IF;
 
-        RAISE NOTICE '%%s updated', let_kind;
+        RAISE NOTICE '%s updated', let_kind;
       ELSE
         -- do nothing
-        RAISE NOTICE 'No %%s - nothing to do', let_kind;
+        RAISE NOTICE 'No %s - nothing to do', let_kind;
       END IF;
 
       IF let_kind='inlet' THEN
@@ -448,7 +448,7 @@ BEGIN
 
   -- catch
   EXCEPTION WHEN OTHERS THEN
-    RAISE NOTICE 'EXCEPTION: %%', SQLERRM;
+    RAISE NOTICE 'EXCEPTION: %', SQLERRM;
     RETURN NEW;
 END; $BODY$
 LANGUAGE plpgsql;

datamodel/app/view/vw_tww_reach.py

@@ -5,7 +5,7 @@
 import argparse
 import os
 
-import psycopg2
+import psycopg
 from pirogue.utils import insert_command, select_columns, table_parts, update_command
 from yaml import safe_load
 
@@ -21,7 +21,7 @@ def vw_tww_reach(pg_service: str = None, extra_definition: dict = None):
     assert pg_service
     extra_definition = extra_definition or {}
 
-    conn = psycopg2.connect(f"service={pg_service}")
+    conn = psycopg.connect(f"service={pg_service}")
     cursor = conn.cursor()
 
     view_sql = """

datamodel/app/view/vw_tww_wastewater_structure.py

@@ -5,7 +5,7 @@
 import argparse
 import os
 
-import psycopg2
+import psycopg
 from pirogue.utils import insert_command, select_columns, table_parts, update_command
 from yaml import safe_load
 
@@ -22,9 +22,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
     assert pg_service
     extra_definition = extra_definition or {}
 
-    variables = {"SRID": int(srid)}
-
-    conn = psycopg2.connect(f"service={pg_service}")
+    conn = psycopg.connect(f"service={pg_service}")
     cursor = conn.cursor()
 
     view_sql = """
@@ -56,7 +54,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
         , main_co_sp.renovation_demand AS co_renovation_demand
 
         , {main_co_cols}
-        , ST_Force2D(COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry))::geometry(Point, %(SRID)s) AS situation3d_geometry
+        , ST_Force2D(COALESCE(wn.situation3d_geometry, main_co.situation3d_geometry))::geometry(Point, {srid}) AS situation3d_geometry
 
         , {ma_columns}
 
@@ -94,6 +92,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
         ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER co_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','cover');
         ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER wn_obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_node');
     """.format(
+        srid=srid,
         extra_cols="\n    ".join(
             [
                 select_columns(
@@ -221,7 +220,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
         ),
     )
 
-    cursor.execute(view_sql, variables)
+    cursor.execute(view_sql)
 
     trigger_insert_sql = """
     CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_wastewater_structure_INSERT()
@@ -438,7 +437,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
         SET situation3d_geometry = ST_SetSRID( ST_MakePoint(
         ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )),
         ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(WN.situation3d_geometry), ST_Y(WN.situation3d_geometry)), dx, dy )),
-        ST_Z(WN.situation3d_geometry)), %(SRID)s )
+        ST_Z(WN.situation3d_geometry)), {srid} )
         WHERE obj_id IN
         (
           SELECT obj_id FROM tww_od.wastewater_networkelement
@@ -450,7 +449,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
         SET situation3d_geometry = ST_SetSRID( ST_MakePoint(
         ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )),
         ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(CO.situation3d_geometry), ST_Y(CO.situation3d_geometry)), dx, dy )),
-        ST_Z(CO.situation3d_geometry)), %(SRID)s )
+        ST_Z(CO.situation3d_geometry)), {srid} )
         WHERE obj_id IN
         (
           SELECT obj_id FROM tww_od.structure_part
@@ -466,7 +465,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
             ST_SetSRID( ST_MakePoint(
                 ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )),
                 ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_PointN(RE.progression3d_geometry, 1)), ST_Y(ST_PointN(RE.progression3d_geometry, 1))), dx, dy )),
-                ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s )
+                ST_Z(ST_PointN(RE.progression3d_geometry, 1))), {srid} )
           ) )
         WHERE fk_reach_point_from IN
         (
@@ -483,7 +482,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
             ST_SetSRID( ST_MakePoint(
                 ST_X(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )),
                 ST_Y(ST_TRANSLATE(ST_MakePoint(ST_X(ST_EndPoint(RE.progression3d_geometry)), ST_Y(ST_EndPoint(RE.progression3d_geometry))), dx, dy )),
-                ST_Z(ST_PointN(RE.progression3d_geometry, 1))), %(SRID)s )
+                ST_Z(ST_PointN(RE.progression3d_geometry, 1))), {srid} )
           ) )
         WHERE fk_reach_point_to IN
         (
@@ -505,6 +504,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
     CREATE TRIGGER vw_tww_wastewater_structure_UPDATE INSTEAD OF UPDATE ON tww_app.vw_tww_wastewater_structure
       FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_wastewater_structure_UPDATE();
     """.format(
+        srid=srid,
         update_co=update_command(
             pg_cur=cursor,
             table_schema="tww_od",
@@ -605,7 +605,7 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
         ),
     )
 
-    cursor.execute(update_trigger_sql, variables)
+    cursor.execute(update_trigger_sql)
 
     trigger_delete_sql = """
     CREATE OR REPLACE FUNCTION tww_app.ft_vw_tww_wastewater_structure_DELETE()
@@ -622,7 +622,8 @@ def vw_tww_wastewater_structure(srid: int, pg_service: str = None, extra_definit
     CREATE TRIGGER vw_tww_wastewater_structure_DELETE INSTEAD OF DELETE ON tww_app.vw_tww_wastewater_structure
       FOR EACH ROW EXECUTE PROCEDURE tww_app.ft_vw_tww_wastewater_structure_DELETE();
     """
-    cursor.execute(trigger_delete_sql, variables)
+
+    cursor.execute(trigger_delete_sql)
 
     extras = """
     ALTER VIEW tww_app.vw_tww_wastewater_structure ALTER obj_id SET DEFAULT tww_sys.generate_oid('tww_od','wastewater_structure');

datamodel/requirements.txt

@@ -1,3 +1,3 @@
-pirogue>=1.4.2
-psycopg2
+pirogue==2.0.1b3
+psycopg>=3.1.18
 pyyaml

datamodel/test/test_geometry.py

@@ -2,8 +2,7 @@
 import os
 import unittest
 
-import psycopg2
-import psycopg2.extras
+import psycopg
 
 from .utils import DEFAULT_PG_SERVICE, DbTestBase
 
@@ -19,7 +18,7 @@ def tearDownClass(cls):
     @classmethod
     def setUpClass(cls):
         pgservice = os.environ.get("PGSERVICE") or DEFAULT_PG_SERVICE
-        cls.conn = psycopg2.connect(f"service={pgservice}")
+        cls.conn = psycopg.connect(f"service={pgservice}")
 
     def test_vw_tww_reach_geometry_insert(self):
         # 1. insert geometry with Z and no rp_from_level and no rp_to_level