diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml new file mode 100644 index 00000000..1cdd5701 --- /dev/null +++ b/.github/workflows/cicd.yaml @@ -0,0 +1,52 @@ +name: CICD + +# on: +# pull_request: +# branches: [main] + +on: push + +jobs: + lidar_on_docker: + runs-on: self-hosted + + steps: + + - name: Checkout branch + uses: actions/checkout@v2 + + - name: build docker image + run: docker build -t lidar_deep_im . + + - name: Check code neatness (linter) + run: docker run lidar_deep_im flake8 + + # - name: unit testing + # run: docker run lidar_deep_im pytest --ignore=actions-runner --ignore="notebooks" + + - name: Full module run on LAS subset + run: docker run -v /var/data/CICD_github_assets:/CICD_github_assets lidar_deep_im + # sudo mount -v -t cifs -o user=mdaab,domain=IGN,uid=24213,gid=10550 //store.ign.fr/store-lidarhd/projet-LHD/IA/Validation_Module/CICD_github_assets/B2V0.5 /var/data/CICD_github_assets + + # - name: Evaluate decisions using optimization code on a single, corrected LAS + # run: > + # docker run -v /var/data/cicd/CICD_github_assets:/CICD_github_assets lidar_deep_im + # python lidar_prod/run.py print_config=true +task='optimize' + # +building_validation.optimization.debug=true + # building_validation.optimization.todo='prepare+evaluate+update' + # building_validation.optimization.paths.input_las_dir=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/20211001_buiding_val_val/ + # building_validation.optimization.paths.results_output_dir=/CICD_github_assets/opti/ + # building_validation.optimization.paths.building_validation_thresholds_pickle=/CICD_github_assets/M8.3B2V0.0/optimized_thresholds.pickle + + - name: chekc the user + run: whoami + + - name: save the docker image because everything worked + run: docker save lidar_deep_im > /var/data/CICD_github_assets/lidar_deep_im.tar # need writing right + + - name: clean the server for further uses + if: always() # always do it, even if something failed + run: docker system prune # remove obsolete docker images (take a HUGE amount of space) + + + diff --git a/.github/workflows/gh-pages b/.github/workflows/gh-pages new file mode 100644 index 00000000..b081bfae --- /dev/null +++ b/.github/workflows/gh-pages @@ -0,0 +1,52 @@ +# Workflow name +name: "Documentation Build" + +# Event that must trigger the workflow +on: + push: # <- trigger when we call push + branches: + - main # <- but only on main branch + - FixDocAPI # <- also on this branch until documentation is up and running. + +jobs: + + build-and-deploy: + runs-on: ubuntu-latest + + # Task to do when launching the workflow + steps: + + # 1. First get the repository source + + - name: "Checkout" + uses: actions/checkout@v2 + + # 2. Sphinx part : install tool and dependencies + + - name: "Set up Python" + uses: actions/setup-python@v1 + with: + python-version: 3.9.12 + + # Packages that depend on torch need need to be installed afterwards, + # hence the "requirements_torch_deps.txt file. + - name: "Install Python dependencies" + working-directory: ./docs/ + run: | + python3 -m pip install --upgrade pip + pip3 install -r requirements.txt + pip3 install -r requirements_torch_deps.txt + + + - name: "Build Sphinx Doc" + working-directory: ./docs/ + run: | + make html + + # 3. Déploiement sur les Github Pages + + - name: "Deploy Github Pages" + uses: JamesIves/github-pages-deploy-action@3.7.1 + with: + BRANCH: gh-pages # <- Branch where generated doc files will be commited + FOLDER: docs/build/html/ # <- Dir where .nojekyll is created and from which to deploy github pages. diff --git a/dockerfile b/dockerfile new file mode 100644 index 00000000..11d0e221 --- /dev/null +++ b/dockerfile @@ -0,0 +1,64 @@ +FROM nvidia/cuda:10.1-devel-ubuntu18.04 +# An nvidia image seems to be necessary for torch-points-kernel. Also, a "devel" image seems required for the same library + +# set the IGN proxy, otherwise apt-get and other applications don't work +ENV http_proxy 'http://192.168.4.9:3128/' +ENV https_proxy 'http://192.168.4.9:3128/' + +# set the timezone, otherwise it asks for it... and freezes +ENV TZ=Europe/Paris +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# all the apt-get installs +RUN apt-get update && apt-get upgrade -y && apt-get install -y \ + software-properties-common \ + wget \ + git \ + libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6 # package needed for anaconda + +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ + && /bin/bash ~/miniconda.sh -b -p /opt/conda \ + && rm ~/miniconda.sh + +ENV PATH /opt/conda/bin:$PATH + +WORKDIR /lidar + +# copy all the data now (because the requirements files are needed for anaconda) +COPY . . + +# install the python packages via anaconda +RUN conda env create -f bash/setup_environment/requirements.yml + +# Make RUN commands use the new environment: +SHELL ["conda", "run", "-n", "lidar_multiclass", "/bin/bash", "-c"] + +# install all the dependencies +RUN conda install -y pytorch=="1.10.1" torchvision=="0.11.2" -c pytorch -c conda-forge \ + && conda install pytorch-lightning==1.5.9 -c conda-forge \ + && pip install torch-scatter -f https://data.pyg.org/whl/torch-1.10.1+cpu.html torch-sparse -f https://data.pyg.org/whl/torch-1.10.1+cpu.html torch-geometric \ + && pip install torch-points-kernels --no-cache \ + && pip install torch torchvision \ + && conda install -y pyg==2.0.3 -c pytorch -c pyg -c conda-forge + +# the entrypoint garanty that all command will be runned in the conda environment +ENTRYPOINT ["conda", \ + "run", \ + "-n", \ + "lidar_multiclass"] + +CMD ["python", \ + "-m", \ + "lidar_multiclass.predict", \ + "--config-path", \ + "/CICD_github_assets/parametres_etape1/.hydra", \ + "--config-name", \ + "predict_config_V1.6.3.yaml", \ + "predict.src_las=/CICD_github_assets/parametres_etape1/test/792000_6272000_subset_buildings.las", \ + "predict.output_dir=/CICD_github_assets/output_etape1", \ + "predict.resume_from_checkpoint=/CICD_github_assets/parametres_etape1/checkpoints/epoch_033.ckpt", \ + "predict.gpus=0", \ + "datamodule.batch_size=10", \ + "datamodule.subtile_overlap=0", \ + "hydra.run.dir=/lidar"] + diff --git a/lidar_multiclass/predict.py b/lidar_multiclass/predict.py index 92d6d16b..ed00688c 100644 --- a/lidar_multiclass/predict.py +++ b/lidar_multiclass/predict.py @@ -62,7 +62,7 @@ def predict(config: DictConfig) -> str: @hydra.main(config_path="../configs/", config_name="config.yaml") def main(config: DictConfig): - f"""See function {predict.__name__}. + """See function predict :meta private: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..65cc39b5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,60 @@ +[metadata] +project_name = "Segmentation Validation Model" +author = "Charles GAYDON" +contact = "charles.gaydon@gmail.com" +license_file = LICENSE +description_file = README.md + + +[isort] +line_length = 99 +profile = black +filter_files = True + + +[flake8] +max_line_length = 99 +show_source = True +format = pylint +ignore = + F401 # Module imported but unused + W504 # Line break occurred after a binary operator + F841 # Local variable name is assigned to but never used + F403 # from module import * + E501 # Line too long + E741 # temp ignore + F405 # temp ignore + W503 # temp ignore + F811 # temp ignore + E266 # temp ignore + E262 # temp ignore + W605 # temp ignore + E722 # temp ignore + F541 # temp ignore + W291 # temp ignore + E401 # temp ignore + E402 # temp ignore + W293 # temp ignore + +exclude = + .git + __pycache__ + data/* + tests/* + notebooks/* + logs/* + /home/MDaab/.local/lib/python3.9/ + /home/MDaab/anaconda3/ + +[tool:pytest] +python_files = tests/* +log_cli = True +markers = + slow +addopts = + --durations=0 + --strict-markers + --doctest-modules +filterwarnings = + ignore::DeprecationWarning + ignore::UserWarning