[WIP] Add AG-64/96 extension

.github/workflows/datamodel-create-dumps.yml

@@ -25,33 +25,41 @@ jobs:
     env:
       COMPOSE_PROFILES: schemaspy
 
+    strategy:
+      matrix:
+        extension: ["", "agxx"]
+
     steps:
       - uses: actions/checkout@v4
 
       - name: Docker build
         run: docker compose up -d --build
+        env:
+          EXTENSION_NAMES: ${{ matrix.extension }}
 
       - name: Initialize container
         run: docker compose exec db init_db.sh wait
 
       - name: Create dumps
-        run: docker compose exec db /src/datamodel/scripts/create-dumps.py
+        run: docker compose exec db /src/datamodel/scripts/create-dumps.py --extension_name=${{ matrix.extension }}
 
       - name: Schemaspy
-        run:  docker compose run schemaspy
+        run: docker compose run schemaspy
+        env:
+          EXTENSION_NAMES: ${{ matrix.extension }}
 
       - name: Docker logs
         if: failure()
         run: docker compose logs db
 
       - uses: actions/upload-artifact@v4
         with:
-          name: datamodel-dumps
+          name: ${{ matrix.extension && format('datamodel-dumps-{0}', matrix.extension) || 'datamodel-dumps' }}
           path: datamodel/artifacts/
           if-no-files-found: error
 
       - uses: actions/upload-artifact@v4
         with:
-          name: datamodel-schemaspy
+          name: ${{ matrix.extension && format('datamodel-schemaspy-{0}', matrix.extension) || 'datamodel-schemaspy' }}
           path: datamodel/schemaspy/
           if-no-files-found: error

.github/workflows/datamodel-test.yml

@@ -28,6 +28,7 @@ jobs:
     strategy:
       matrix:
         psycopg-version: [ 2, 3 ]
+        extension: ["", "agxx"]
       fail-fast: true
 
     steps:
@@ -43,7 +44,7 @@ jobs:
 
       - name: Initialize container
         run: |
-          docker run -d -p 5432:5432 --name teksi-wastewater teksi/wastewater:unstable
+          docker run -d -p 5432:5432 --name teksi-wastewater -e EXTENSION_NAMES=${{ matrix.extension }} teksi/wastewater:unstable
           docker exec teksi-wastewater init_db.sh wait
 
       - name: Run tests

.github/workflows/plugin-test.yml

@@ -30,6 +30,7 @@ jobs:
       fail-fast: false
       matrix:
         qgis_version: [3.34-jammy, latest]
+        extension: ["", "agxx"]
     env:
       QGIS_TEST_VERSION: ${{ matrix.qgis_version }}
       COMPOSE_PROFILES: qgis
@@ -57,12 +58,16 @@ jobs:
 
       - name: Clear Database
         run: docker compose exec db init_db.sh build
+        env:
+          EXTENSION_NAMES: ${{ matrix.extension }}
 
       - name: Test on QGIS DSS dataset
         run: docker compose run qgis /usr/src/plugin/.docker/run-docker-tests.sh teksi_wastewater/tests/test_interlis.py::TestInterlis::test_dss_dataset_import_export
 
       - name: Clear Database
         run: docker compose exec db init_db.sh build
+        env:
+          EXTENSION_NAMES: ${{ matrix.extension }}
 
       - name: Test command line import orgs
         run: docker compose run qgis sh -c 'xvfb-run /usr/src/plugin/tww_cmd.py interlis_import --xtf_file /usr/src/plugin/teksi_wastewater/tests/data/minimal-dataset-organisation-arbon-only.xtf --pghost db --pgdatabase tww --pguser postgres --pgpass postgres --pgport 5432'

README.md

@@ -1,4 +1,5 @@
 # TEKSI Wastewater module (TWW)
+
 NEW TEKSI wastewater module, adapted datamodel to fit Swiss Norm VSA-DSS Release 2020.1 new standard.
 
 This repository holds all code related to the TEKSI Wastewater module (formerly QGEP). <!---  // skip-keyword-check -->
@@ -10,7 +11,6 @@ It contains:
  - A QGIS .qgs project : TEKSI wastewater and General Wastewater Management Plan (GWMP)
  - The documentation : https://teksi.github.io/wastewater/
 
-This will replace actual [QGIS QGEP Project](https://github.com/qgep) <!---  // skip-keyword-check -->
 
 How to start [testing](https://github.com/teksi/wastewater/discussions/72)
 

datamodel/app/create_app.py

@@ -9,6 +9,7 @@
 except ImportError:
     import psycopg2 as psycopg
 
+from extensions.extension_manager import load_extension
 from pirogue import MultipleInheritance, SimpleJoins, SingleInheritance
 from triggers.set_defaults_and_triggers import set_defaults_and_triggers
 from view.vw_tww_additional_ws import vw_tww_additional_ws
@@ -45,6 +46,7 @@ def create_app(
     tww_wastewater_structure_extra: Optional[Path] = None,
     tww_ii_extra: Optional[Path] = None,
     wastewater_structure_extra: Optional[Path] = None,
+    extension_names: Optional[list] = [],
 ):
     """
     Creates the schema tww_app for TEKSI Wastewater & GEP
@@ -75,6 +77,10 @@ def create_app(
     run_sql_file("functions/organisation_functions.sql", pg_service, variables)
     run_sql_file("functions/meta_functions.sql", pg_service, variables)
 
+    if extension_names:
+        for extension in extension_names:
+            load_extension(srid, pg_service, "tww", extension)
+
     # open YAML files
     if tww_reach_extra:
         tww_reach_extra = safe_load(open(tww_reach_extra))
@@ -271,6 +277,13 @@ def create_app(
         default=False,
         action=BooleanOptionalAction,
     )
+    parser.add_argument(
+        "-x",
+        "--extension_names",
+        nargs="*",
+        required=False,
+        help="extensions that should be loaded into application schema",
+    )
     args = parser.parse_args()
 
     create_app(
@@ -279,4 +292,5 @@ def create_app(
         drop_schema=args.drop_schema,
         tww_reach_extra=args.tww_reach_extra,
         tww_wastewater_structure_extra=args.tww_wastewater_structure_extra,
+        extension_names=args.extension_names,
     )

datamodel/app/extensions/config.yaml

@@ -0,0 +1,9 @@
+extensions:
+  - id: template
+    directory: template_extension
+    variables:
+      myVariable: "Hello World"
+      myNumber: 0
+  - id: agxx
+    directory: tww2ag6496
+    variables: {}

datamodel/app/extensions/demo_extension/001_demo.sql

@@ -0,0 +1 @@
+SELECT 1;

datamodel/app/extensions/demo_extension/002_demo.py

@@ -0,0 +1,17 @@
+from argparse import ArgumentParser
+
+
+def demo(myVariable, myNumber):
+    print(f"{myVariable}! My favorite number is {myNumber}")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("-v", "--myVariable", help="my variable")
+    parser.add_argument("-n", "--myNumber", help="my number")
+    args = parser.parse_args()
+
+    demo(
+        args.myVariable,
+        args.myNumber,
+    )

datamodel/app/extensions/extension_manager.py

@@ -0,0 +1,118 @@
+import os
+import subprocess
+import sys
+from argparse import ArgumentParser
+from pathlib import Path
+
+from yaml import safe_load
+
+try:
+    import psycopg
+except ImportError:
+    import psycopg2 as psycopg
+
+
+def read_config(config_file: str, extension_name: str):
+    abs_file_path = Path(__file__).parent.resolve() / config_file
+    with open(abs_file_path) as file:
+        config = safe_load(file)
+    for entry in config.get("extensions", []):
+        if entry.get("id") == extension_name:
+            return entry
+    raise ValueError(f"No entry found with id: {extension_name}")
+
+
+def run_py_file(file_path: str, variables: dict = None):
+    abs_file_path = Path(__file__).parent.resolve() / file_path
+    varlist = [sys.executable, str(abs_file_path)]
+    for key, value in variables.iteritems():
+        varlist.append("--" + key)
+        varlist.append(value)
+    result = subprocess.run(varlist, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise RuntimeError(f"Error running file: {result.stderr}")
+
+
+def run_sql_file(file_path: str, pg_service: str, variables: dict = None):
+    abs_file_path = Path(__file__).parent.resolve() / file_path
+    with open(abs_file_path) as f:
+        sql = f.read()
+    run_sql(sql, pg_service, variables)
+
+
+def run_sql(sql: str, pg_service: str, variables: dict = None):
+    if variables:
+        sql = psycopg.sql.SQL(sql).format(**variables)
+    conn = psycopg.connect(f"service={pg_service}")
+    cursor = conn.cursor()
+    cursor.execute(sql)
+    conn.commit()
+    conn.close()
+
+
+def load_extension(
+    srid: int = 2056,
+    pg_service: str = "pg_tww",
+    module_name: str = "tww",
+    extension_name: str = None,
+):
+    """
+    initializes the TWW database for usage of an extension
+
+    Args:
+        pgservice: pg service string
+        extension_name: Name of the extension to load
+
+    """
+
+    # load definitions from config
+    config = read_config("config.yaml", extension_name)
+    variables = config.get("variables", {})
+
+    # We also disable symbology triggers as they can badly affect performance. This must be done in a separate session as it
+    # would deadlock other sessions.
+    conn = psycopg.connect(f"service={pg_service}")
+    cursor = conn.cursor()
+    cursor.execute(f"SELECT {module_name}_app.alter_symbology_triggers('disable');")
+    conn.commit()
+    conn.close()
+
+    directory = config.get("directory", None)
+    if directory:
+        files = os.listdir(Path(__file__).parent.resolve() / directory)
+        files.sort()
+        for file in files:
+            filename = os.fsdecode(file)
+            if filename.lower().endswith(".sql"):
+                print(f"Running {filename}")
+                run_sql_file(os.path.join(directory, filename), pg_service, variables)
+            if filename.endswith(".py"):
+                print(f"Running {filename}")
+                run_py_file(os.path.join(directory, filename), variables)
+
+    # re-create symbology triggers
+    conn = psycopg.connect(f"service={pg_service}")
+    cursor = conn.cursor()
+    cursor.execute(f"SELECT {module_name}_app.alter_symbology_triggers('enable');")
+    conn.commit()
+    conn.close()
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("-p", "--pg_service", help="postgres service")
+    parser.add_argument("-m", "--module_name", help="name of TEKSI module", default="tww")
+    parser.add_argument(
+        "-s", "--srid", help="SRID EPSG code, defaults to 2056", type=int, default=2056
+    )
+    parser.add_argument(
+        "-x", "--extension_name", help="name of the database extension", type=str, default="demo"
+    )
+    args = parser.parse_args()
+
+    load_extension(
+        args.srid,
+        args.pg_service,
+        args.module_name,
+        args.extension_name,
+    )

datamodel/app/extensions/tww2ag6496/002_extensions_cfg_agxx_sys.sql

@@ -0,0 +1,76 @@
+-- Funktion zum Mapping der Organisations-ID
+
+CREATE OR REPLACE FUNCTION tww_app.fct_agxx_organisationid_to_vsa(oid varchar)
+RETURNS varchar(16)
+AS
+$BODY$
+DECLARE
+	out_oid varchar(16);
+BEGIN
+	SELECT obj_id INTO out_oid FROM tww_od.organisation WHERE right(obj_id,8) = right(oid,8);
+	return out_oid;
+END;
+$BODY$
+LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION tww_app.fct_agxx_linkup_secondary_nodes(oid varchar)
+RETURNS VOID
+AS
+$BODY$
+DECLARE
+DECLARE
+	ws_oid varchar(16);
+	wn_oids varchar(16)[];
+	sp_oids varchar(16)[];
+BEGIN
+     SELECT ws.obj_id, array_agg(wn.obj_id) INTO  ws_oid, wn_oids
+	  FROM (SELECT ws.obj_id,
+			st_buffer(detail_geometry3d_geometry,0.0001)
+			as detail_geometry2d_geometry
+		 FROM tww_od.wastewater_structure ws
+		 LEFT JOIN tww_od.channel ch on ws.obj_id=ch.obj_id
+		 WHERE ws.detail_geometry3d_geometry IS NOT NULL
+		 AND ch.obj_id is NULL) ws
+	  LEFT JOIN  (SELECT obj_id,situation3d_geometry
+				  from tww_od.wastewater_node) wn ON
+		ST_CoveredBy(wn.situation3d_geometry
+		  , detail_geometry2d_geometry)
+	  LEFT JOIN (SELECT obj_id,fk_wastewater_structure FROM tww_od.wastewater_networkelement) ne on ne.obj_id=wn.obj_id
+	  WHERE ne.fk_wastewater_structure IS NULL
+	  GROUP BY ws.obj_id;
+
+	SELECT array_agg(co.obj_id) INTO sp_oids
+	  FROM tww_od.wastewater_node wn
+	  LEFT JOIN tww_od.cover co on co.ag64_fk_wastewater_node=wn.obj_id
+	  WHERE wn.obj_id = any(wn_oids)
+	  GROUP BY wn.obj_id;
+
+	CASE WHEN ws_oid is not NULL THEN
+	  UPDATE tww_od.wastewater_networkelement
+        SET fk_wastewater_structure = ws_oid
+        WHERE obj_id = wn_oid;
+
+      UPDATE tww_od.structure_part
+        SET fk_wastewater_structure = ws_oid
+        WHERE obj_id = ANY(sp_oids);
+
+	ELSE NULL;
+	END CASE;
+	RETURN;
+END;
+$BODY$
+LANGUAGE plpgsql;
+
+-- wird für Updates von letzte_aenderung_wi/gep genutzt
+CREATE TABLE IF NOT EXISTS tww_cfg.agxx_last_modification_updater(
+	username varchar(200) PRIMARY KEY
+	, ag_update_type varchar(4) NOT NULL
+	, CONSTRAINT ag_update_type_valid CHECK (ag_update_type = ANY(ARRAY['None','WI','GEP','Both'])));
+
+------------------
+-- System tables
+------------------
+INSERT INTO tww_sys.dictionary_od_table (id, tablename, shortcut_en) VALUES
+(2999998,'measure_text','MX'),
+(2999999,'building_group_text','GX')
+ON CONFLICT DO NOTHING;