Add Building Completion

.github/workflows/cicd.yaml

@@ -13,7 +13,7 @@ jobs:
     steps:
 
     - name: Checkout branch       
-      uses: actions/checkout@v2
+      uses: actions/checkout@v1
 
     - name: build docker image       
       run: docker build -t lidar_prod_im .
@@ -24,7 +24,10 @@ jobs:
     - name: unit testing
       run: docker run lidar_prod_im pytest --ignore=actions-runner --ignore="notebooks"
 
-    - name: Evaluate decisions on single LAS
+    - name: Full module run on LAS subset
+      run: docker run -v /var/data/cicd/CICD_github_assets:/CICD_github_assets lidar_prod_im 
+
+    - name: Evaluate decisions using optimization code on a single, corrected LAS 
       run: >
         docker run -v /var/data/cicd/CICD_github_assets:/CICD_github_assets lidar_prod_im
         python lidar_prod/run.py print_config=true +task='optimize'
@@ -34,9 +37,6 @@ jobs:
         building_validation.optimization.paths.results_output_dir=/CICD_github_assets/opti/
         building_validation.optimization.paths.building_validation_thresholds_pickle=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/M8.0B2V0.0_buildingvalidation_thresholds.pickle
 
-    - name: test full run
-      run: docker run -v /var/data/cicd/CICD_github_assets:/CICD_github_assets predict_im 
-
     - name: clean the server for further uses
       if: always()  # always do it, even if something failed
       run: docker system prune  # remove obsolete docker images (take a HUGE amount of space)

README.md

@@ -47,7 +47,19 @@ Decision thresholds `C1`, `C2`, `R1`, `R2`, `O1` are chosen via a multi-objectiv
 
 ![](assets/img/LidarBati-BuildingValidationM7.1V2.0.png)
 
-#### B) Building Identification
+#### B) Building Completion
+
+Goal: Confirm points that were too isolated to make up a group but have high-enough probability nevertheless (e.g. walls)
+
+Identify  _candidate buildings points_ that have not been clustered in previous step due AND have high enough probability (p>=0.5)).
+Cluster them together with previously confirmed building points in a relaxed, vertical fashion (higher tolerance, XY plan).
+For each cluster, if some points were confirmed, the others are considered to belong to the same building, and are 
+therefore confirmed as well.
+
+![](assets/img/LidarBati-BuildingCompletion.png)
+
+
+#### C) Building Identification
 
 Goal: Highlight potential buildings that were missed by the rule-based algorithm, for human inspection. 
 
@@ -92,7 +104,7 @@ pip install --upgrade https://github.com/IGNF/lidar-prod-quality-control/tarball
 pip install -e .  # from local sources
 ```
 
-To run the module as a package, you will need a source cloud point in LAS format with an additional channel containing predicted building probabilities. The name of this channel is specified by `config.data_format.las_channel_names.ai_building_proba`.
+To run the module as a package, you will need a source cloud point in LAS format with an additional channel containing predicted building probabilities. The name of this channel is specified by `config.data_format.las_dimensions.ai_building_proba`.
 
 To run using default configurations of the installed package, use
 ```bash

configs/building_completion/default.yaml

@@ -0,0 +1,15 @@
+_target_: lidar_prod.tasks.building_completion.BuildingCompletor
+
+data_format: ${data_format}
+
+# TODO: uncomment when min_building_proba_relaxation_if_bd_uni_overlay is specified <1 to be consistent everywhere.
+# min_building_proba: ${building_validation.application.rules.min_confidence_confirmation}
+
+min_building_proba: 0.5
+min_building_proba_relaxation_if_bd_uni_overlay: 1.0  # No effect if = 1.0
+
+cluster:
+  min_points: 10 # including isolated points (in BuildingValidator) and confirmed candidates points.
+  tolerance: 1  # meters
+  is3d: false  # group in 2d for better detection
+

configs/building_identification/default.yaml

@@ -1,14 +1,12 @@
 _target_: lidar_prod.tasks.building_identification.BuildingIdentifier
 
-candidate_buildings_codes: ${data_format.codes.candidates.building}
-
 data_format: ${data_format}
 
 min_building_proba: ${building_validation.application.rules.min_confidence_confirmation}
-min_building_proba_multiplier_if_bd_uni_overlay: 1.0
+min_building_proba_relaxation_if_bd_uni_overlay: 1.0
 
 cluster:
-  min_points: 200
+  min_points: 200  # Large so that small artefact are ignored
   tolerance: 1  # meters
   is3d: false  # group in 2d for better detection
 

configs/building_validation/application/default.yaml

@@ -3,7 +3,6 @@ _target_: lidar_prod.tasks.building_validation.BuildingValidator
 building_validation_thresholds_pickle: /path/to/best_trial.pkl
 
 data_format: ${data_format}
-candidate_buildings_codes: ${data_format.codes.candidates.building}
 use_final_classification_codes: true
 
 
@@ -27,18 +26,3 @@ rules:
   min_uni_db_overlay_frac: 0.508
   min_confidence_refutation: 0.872
   min_frac_refutation: 0.964
-
-codes:
-  detailed:
-    unclustered: 202  # refuted
-    ia_refuted: 110  # refuted
-    ia_refuted_and_db_overlayed: 111  # unsure
-    both_unsure: 112  # unsure
-    ia_confirmed_only: 113  # confirmed
-    db_overlayed_only: 114  # confirmed
-    both_confirmed: 115  # confirmed
-  final:
-    unsure: 214  # unsure
-    not_building: 208  # refuted
-    building: 6  # confirmed
-

configs/config.yaml

@@ -8,4 +8,5 @@ defaults:
   - data_format: default.yaml
   - building_validation: default.yaml
   - building_identification: default.yaml
+  - building_completion: default.yaml
   - _self_  # needed by pdal for legacy reasons

configs/data_format/cleaning/default.yaml

@@ -3,5 +3,5 @@ _target_: lidar_prod.tasks.cleaning.Cleaner
 # Extra dims that are kept when cleaning dimensions
 # You can override with "all" to keep all extra dimensions at development time.
 keep_extra_dims:
-  - "${data_format.las_channel_names.ai_building_identified}=uint"
-  - "${data_format.las_channel_names.ai_building_proba}=float"
+  - "${data_format.las_dimensions.ai_building_identified}=uint"
+  - "${data_format.las_dimensions.ai_building_proba}=float"

configs/data_format/default.yaml

@@ -2,22 +2,38 @@ tile_size_meters: 1000
 crs_prefix: "EPSG:"
 crs: 2154
 
-# To connect logic between application tasks
-las_channel_names:
+# Those names connect the logics between successive tasks
+las_dimensions:
   # input
   classification: classification  #las format
-  cluster_id: ClusterID  #pdal-defined
-  uni_db_overlay: BDTopoOverlay  # user-defined
-  ai_building_proba: building  # user-defined
+  ai_building_proba: building  # user-defined - output by deep learning model
 
-  # custom channels
-  candidate_buildings_flag: CandidateBuildingsFlag
-  macro_candidate_building_groups: CandidateBuildingGroups
+  # intermediary channels
+  cluster_id: ClusterID  # pdal-defined -> created by clustering operations
+  uni_db_overlay: BDTopoOverlay  # user-defined -> a 0/1 flag for presence of a BDUni vector
+  candidate_buildings_flag: F_CandidateB # -> a 0/1 flag identifying candidate buildings found by rules-based classification  
+  ClusterID_candidate_building: CID_CandidateB  # -> Cluster index from BuildingValidator, 0 if no cluster, 1-n elsewise
+  ClusterID_isolated_plus_confirmed: CID_IsolatedOrConfirmed  # -> Cluster index from BuildingCompletor, 0 if no cluster, 1-n elsewise
+
+
+  # additionnal output channel
   ai_building_identified: Group
 
 codes:
-  candidates:
-    building: [202]
+  building:
+    candidates: [202]  # found by rules-based classification (TerraScan)
+    detailed:  # used for detailed output when doing threshold optimization
+      unclustered: 202  # refuted
+      ia_refuted: 110  # refuted
+      ia_refuted_and_db_overlayed: 111  # unsure
+      both_unsure: 112  # unsure
+      ia_confirmed_only: 113  # confirmed
+      db_overlayed_only: 114  # confirmed
+      both_confirmed: 115  # confirmed
+    final:  # used at the end of the building process
+      unsure: 214  # unsure
+      not_building: 208  # refuted
+      building: 6  # confirmed
 
 defaults:
   - cleaning: default.yaml

dockerfile

@@ -10,12 +10,12 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # all the apt-get installs
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \
-        software-properties-common  \
-        wget                        \
-        git                         \
-        postgis                     \
-        pdal                        \
-        libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6   # package needed for anaconda
+    software-properties-common  \
+    wget                        \
+    git                         \
+    postgis                     \
+    pdal                        \
+    libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6   # package needed for anaconda
 
 # install anaconda
 RUN wget --quiet https://repo.anaconda.com/archive/Anaconda3-2021.11-Linux-x86_64.sh -O ~/anaconda.sh
@@ -40,15 +40,15 @@ RUN python -c "import pdal"
 
 # the entrypoint garanty that all command will be runned in the conda environment
 ENTRYPOINT ["conda",                \   
-            "run",                  \
-            "-n",                   \
-            "lidar_prod"]
+    "run",                  \
+    "-n",                   \
+    "lidar_prod"]
 
 # cmd for a normal run (non evaluate)
 CMD        ["python",               \
-            "lidar_prod/run.py",    \
-            "print_config=true",    \
-            "paths.src_las=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/subsets/871000_6617000_subset_with_probas.las", \
-            "paths.output_dir=/CICD_github_assets/app/", \
-            "data_format.codes.candidates.building=[19, 20, 110, 112, 114, 115]", \
-            "building_validation.application.building_validation_thresholds_pickle=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/M8.0B2V0.0_buildingvalidation_thresholds.pickle"]
+    "lidar_prod/run.py",    \
+    "print_config=true",    \
+    "paths.src_las=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/subsets/871000_6617000_subset_with_probas.las", \
+    "paths.output_dir=/CICD_github_assets/app/", \
+    "data_format.codes.building.candidates=[202]", \
+    "building_validation.application.building_validation_thresholds_pickle=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/M8.0B2V0.0_buildingvalidation_thresholds.pickle"]

lidar_prod/application.py

@@ -1,9 +1,11 @@
 import logging
 import os
 import os.path as osp
+from tempfile import TemporaryDirectory
 import hydra
 from omegaconf import DictConfig
 from typing import Optional
+from lidar_prod.tasks.building_completion import BuildingCompletor
 from lidar_prod.tasks.cleaning import Cleaner
 
 from lidar_prod.utils import utils
@@ -30,13 +32,20 @@ def apply(config: DictConfig):
     in_f = config.paths.src_las
     out_f = osp.join(config.paths.output_dir, osp.basename(in_f))
 
-    bv: BuildingValidator = hydra.utils.instantiate(
-        config.building_validation.application
-    )
-    bv.run(in_f, out_f)
+    with TemporaryDirectory() as td:
+        # Temporary LAS file for intermediary results.
+        temp_f = osp.join(td, osp.basename(in_f))
 
-    bi: BuildingIdentifier = hydra.utils.instantiate(config.building_identification)
-    bi.run(out_f, out_f)
+        bv: BuildingValidator = hydra.utils.instantiate(
+            config.building_validation.application
+        )
+        bv.run(in_f, temp_f)
 
-    cl: Cleaner = hydra.utils.instantiate(config.data_format.cleaning)
-    cl.run(out_f, out_f)
+        bc: BuildingCompletor = hydra.utils.instantiate(config.building_completion)
+        bc.run(temp_f, temp_f)
+
+        bi: BuildingIdentifier = hydra.utils.instantiate(config.building_identification)
+        bi.run(temp_f, temp_f)
+
+        cl: Cleaner = hydra.utils.instantiate(config.data_format.cleaning)
+        cl.run(temp_f, out_f)