[WIP] Add Docs

.github/workflows/cicd.yaml

@@ -1,9 +1,18 @@
 name: CICD
 
 on:
+  # Run CICD for non-draft pull request
   pull_request:
-    branches: [dev, main]
-
+    branches:
+      - dev
+      - main
+  # Also run when the pull request merges (which generates a push)
+  # So that we can tag the docker image appropriately.
+  push:
+    branches:
+      - dev
+      - prod
+      - main
 
 jobs:
   lidar_on_docker:
@@ -20,20 +29,22 @@ jobs:
     - name: Check code neatness (linter)
       run: docker run lidar_prod_im flake8
 
-    - name: Run tests - fast ones go first.
-      run: docker run lidar_prod_im python -m pytest -v -m "not slow" --ignore=actions-runner
+    - name: Run tests & get coverage - fast ones go first.
+      run: >
+        docker run
+        lidar_prod_im
+        python -m
+        pytest -rA -v -m "not slow" --ignore=actions-runner
 
-    - name: Run slow tests last.
+    - name: Run slow tests last (evaluation on large file)
       run: >
-        docker run 
+        docker run
         -v /var/data/cicd/CICD_github_assets/M8.4/inputs/evaluation/:/lidar/tests/files/large/
         lidar_prod_im
         python -m
-        pytest -v -m slow
-        --ignore=actions-runner
-        --no-cov
+        pytest -rA -v -m "slow" --ignore=actions-runner --no-cov
 
-    - name: Full module run on LAS subset using CLI.
+    - name: Test run from CLI on a LAS subset.
       run: >
         docker run 
         -v /var/data/cicd/CICD_github_assets/M8.4/inputs/:/inputs/
@@ -45,22 +56,26 @@ jobs:
         paths.src_las=/inputs/730000_6360000.subset.prototype_format202.las
         paths.output_dir=/outputs/
 
-    # This is somewhat redundant with unit test but may serve a a doc on usage ?
-    - name: Evaluate decisions using optimization task (debug mode means on a single, corrected LAS) using CLI.
-      run: >
-        docker run 
-        -v /var/data/cicd/CICD_github_assets/M8.4/inputs/evaluation/:/inputs/
-        -v /var/data/cicd/CICD_github_assets/M8.4/outputs/evaluation/:/outputs/ lidar_prod_im
-        python lidar_prod/run.py 
-        print_config=true 
-        +task='optimize'
-        +building_validation.optimization.debug=true
-        building_validation.optimization.todo='prepare+evaluate+update'
-        building_validation.optimization.paths.input_las_dir=/inputs/
-        building_validation.optimization.paths.results_output_dir=/outputs/
-        building_validation.optimization.paths.building_validation_thresholds_pickle=/inputs/optimized_thresholds.pickle
-
-    - name: clean the server for further uses
+    # Everything ran so we tag the valid docker image to keep it
+    # This happens for push events, which are in particular 
+    # triggered when a pull request is merged.
+    - name: Get the branch name into an environment variable.
+      if: github.event_name == 'push'
+      uses: nelonoel/[email protected]
+
+    - name: Print the branch name
+      if: github.event_name == 'push'
+      run: echo ${BRANCH_NAME}
+
+    - name: Tag the docker image with branch name
+      if: github.event_name == 'push'
+      run: docker tag lidar_prod_im:latest lidar_prod_im:${BRANCH_NAME}
+
+    - name: Dry run image so that is it not prunned
+      if: github.event_name == 'push'
+      run: docker run lidar_prod_im:${BRANCH_NAME} bash
+
+    - name: Clean dangling docker images
       if: always()  # always do it, even if something failed
       run: docker system prune  # remove obsolete docker images (take a HUGE amount of space)
 

.github/workflows/gh-pages.yaml

@@ -0,0 +1,68 @@
+# Workflow name
+name: "Documentation Build"
+
+# Event that must trigger the workflow
+on:
+  push:
+    branches:
+      - main  # <- only on main branch
+      - add-documentation  # <- on this branch until documentation is merged once in main.
+
+jobs:
+
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    # Use bash instead of sh for conda activation
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      # Checkout the repository
+      - name: "Checkout"
+        uses: actions/checkout@v2
+
+      # See https://github.com/conda-incubator/setup-miniconda#caching-environments
+
+      # Setup empty conda environment
+      - name: Setup a conda-incubator with an empty conda env
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            python-version: 3.9.12
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            use-mamba: true
+            # Environment to create and activate for next steps
+            activate-environment: lidar_prod
+
+      # Cache the env
+      # See https://github.com/conda-incubator/setup-miniconda#caching-environments
+      - name: Get Date
+        id: get-date
+        run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
+        shell: bash
+
+      - name: Cache conda environment
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-${{ runner.os }}--${{ runner.arch }}--${{ steps.get-date.outputs.today }}-${{ hashFiles('bash/setup_environment/requirements.yml') }}-${{ env.CACHE_NUMBER }}
+        id: cache
+
+      - name: Update environment if there was no cached env.
+        run: mamba env update -n lidar_prod -f bash/setup_environment/requirements.yml
+        if: steps.cache.outputs.cache-hit != 'true'
+
+      # 2. Sphinx part : install tool and dependencies
+      - name: "Build Sphinx Doc"
+        working-directory: ./docs/
+        run: make html
+
+      # 3. Déploiement sur les Github Pages
+
+      - name: "Deploy Github Pages"
+        uses: JamesIves/[email protected]
+        with:
+          BRANCH: gh-pages     # <- Branch where generated doc files will be commited
+          FOLDER: ./docs/build/html/  # <- Dir where .nojekyll is created and from which to deploy github pages.

Dockerfile

@@ -1,49 +1,41 @@
-FROM nvidia/cuda:11.3.1-devel-ubuntu20.04
+FROM mambaorg/micromamba:latest
 
 # set the IGN proxy, otherwise apt-get and other applications don't work 
+# from within our self-hoster action runner
 ENV http_proxy 'http://192.168.4.9:3128/'
 ENV https_proxy 'http://192.168.4.9:3128/'
 
-# set the timezone, otherwise it asks for it... and freezes
-ENV TZ=Europe/Paris
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
 # all the apt-get installs
+USER root
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \
-    software-properties-common  \
-    wget                        \
-    git                         \
-    postgis                     \
-    libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6   # package needed for anaconda
-
-# install anaconda
-RUN wget --quiet https://repo.anaconda.com/archive/Anaconda3-2021.11-Linux-x86_64.sh -O ~/anaconda.sh
-RUN /bin/bash ~/anaconda.sh -b -p /opt/conda && \
-    rm ~/anaconda.sh
-ENV PATH /opt/conda/bin:$PATH
+    software-properties-common \
+    wget \
+    git \
+    postgis                 
 
-
-# /lidar becomes the working directory, where the repo content is copied.
+# /lidar becomes the working directory, where the repo content 
+# (where this Dockerfile lives) is copied.
 WORKDIR /lidar
 COPY . .
 
-# install the python packages via anaconda
-RUN conda env create -f bash/setup_environment/requirements.yml
-
-# Make RUN commands use the new environment:
-SHELL ["conda", "run", "-n", "lidar_prod", "/bin/bash", "-c"]
+# Copy requirements so that pip installs can occur smootly. 
+COPY bash/setup_environment/requirements.yml /tmp/env.yaml
+COPY bash/setup_environment/requirements.txt /tmp/requirements.txt
 
+# install the python packages via anaconda
+RUN micromamba create --yes --file /tmp/env.yaml
+# Sets the environment name since it is not "base"
+# This ensure that it is activate when usign "docker run ..."
+ENV ENV_NAME lidar_prod
+# Make RUN commands here use the environment
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+# List packages and their version
+RUN micromamba list
 # test if pdal is installed (a tricky library!)
 RUN echo "Make sure pdal is installed:"
 RUN python -c "import pdal"
 
-# the entrypoint garanties that all command will be runned in the conda environment
-ENTRYPOINT ["conda", \
-    "run", \
-    "-n", \
-    "lidar_prod"]
-
-# cmd for a normal run (non evaluate)
+# Example command to run the application from within the image
 CMD  ["python", \
     "lidar_prod/run.py", \
     "print_config=true", \