Improve documentation (#70)

Co-authored-by: Daniel Morcuende <[email protected]>

* General improvements
* Adds example notebooks

.gitattributes

@@ -0,0 +1 @@
+*.py diff=python

.github/workflows/ci.yml

@@ -99,11 +99,34 @@ jobs:
         run: |
           python -m pip install --upgrade pip
 
+      - name: Install pandoc
+        run: |
+          sudo apt install pandoc
+
       - name: Install doc dependencies
         run: |
           pip install -e .[doc]
           python -c 'import iop4lib; print(iop4lib.__version__)'
 
-      - name: Build the docs
-        run: | 
-          make docs-sphinx
+      - name: Download test data
+        env:
+          TEST_DATA_PASSWORD: ${{ secrets.test_data_password }}
+        run: |
+          export TESTDATA_MD5SUM=`grep 'TESTDATA_MD5SUM' ./tests/conftest.py | awk -F"'" '{print $2}' | tr -d '\n'`
+          wget --post-data "pass=$TEST_DATA_PASSWORD" "https://vhega.iaa.es/iop4/iop4testdata.tar.gz?md5sum=$TESTDATA_MD5SUM" -O $HOME/iop4testdata.tar.gz
+
+      - name: Extract test data in the default data folder
+        run: |
+          tar -xzf $HOME/iop4testdata.tar.gz -C $HOME
+          mv $HOME/iop4testdata $HOME/.iop4data  
+
+      - name: Create the DB
+        run: |
+          cd iop4site
+          python manage.py makemigrations
+          python manage.py makemigrations iop4api
+          python manage.py migrate --no-input
+          ls -lh $HOME/.iop4data/
+
+      - name: Build docs
+        run: make docs-sphinx

CITATION.cff

@@ -0,0 +1,38 @@
+cff-version: 1.2.0
+title: IOP4
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file.
+type: software
+authors:
+  - given-names: Juan
+    family-names: Escudero Pedrosa
+    email: [email protected]
+    affiliation: IAA-CSIC
+    orcid: 'https://orcid.org/0000-0002-4131-655X'
+  - given-names: Daniel
+    family-names: Morcuende Parrilla
+    email: [email protected]
+    affiliation: IAA-CSIC
+    orcid: 'https://orcid.org/0000-0001-9400-0922'
+  - given-names: Jorge
+    family-names: Otero-Santos
+    email: [email protected]
+    affiliation: IAA-CSIC
+    orcid: 'https://orcid.org/0000-0002-4241-5875'
+identifiers:
+  - type: doi
+    value: 10.5281/zenodo.10222722
+repository-code: 'https://github.com/juanep97/iop4'
+url: 'https://vhega.iaa.es/iop4/about/'
+abstract: IOP4, the Interactive Optical Photo-Polarimetric Python Pipeline.
+keywords:
+  - astronomy
+  - photometry
+  - polarimetry
+  - database
+  - pipeline
+  - python
+  - interactive
+  - blazars
+license: BSD-3-Clause

Makefile

@@ -25,4 +25,4 @@ docs-sphinx:
 	cd docs && python -m sphinx . _build/html -b html
 
 docs-show:
-	open docs/_build/html/index.html
+	open docs/_build/html/index.html

README.md

@@ -1,5 +1,8 @@
-<a href="https://github.com/juanep97/iop4/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/juanep97/iop4/actions/workflows/ci.yml/badge.svg"/></a>
-<a href="https://zenodo.org/doi/10.5281/zenodo.10222722"><img src="https://zenodo.org/badge/636786270.svg" alt="DOI"></a>
+<div>
+<a href="https://github.com/juanep97/iop4/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/juanep97/iop4/actions/workflows/ci.yml/badge.svg"></img></a>
+<a href="https://zenodo.org/doi/10.5281/zenodo.10222722"><img src="https://zenodo.org/badge/636786270.svg" alt="DOI"></img></a>
+</div>
+
 
 **IOP4** is a complete rewrite of IOP3, a pipeline to work with **photometry** and **polarimetry** of **optical data** from [CAHA](https://www.caha.es/es/) and [OSN](https://www.osn.iaa.csic.es/) observatories. It is built to ease debugging and inspection of data.
 
@@ -18,7 +21,7 @@ If you don't have Python 3.10 or later, you can install [pyenv](https://github.c
     $ curl https://pyenv.run | bash
 ```
 
-Follow the instruction that these command output to add `pyenv`to `PATH` (or copy the commands from https://github.com/pyenv/pyenv for your shell). Restart your terminal, or source the file (e.g. `. ~/.bashrc` or `. ~/.zshrc`) Then, run 
+Follow the instruction that these command output to add `pyenv` to `PATH` (or copy the commands from https://github.com/pyenv/pyenv for your shell). Restart your terminal, or source the file (e.g. `. ~/.bashrc` or `. ~/.zshrc`) Then, run 
 ```bash 
     $ pyenv install 3.10
     $ pyenv virtualenv 3.10 iop4-venv
@@ -138,11 +141,37 @@ You can get an IPython interactive terminal after running iop4 using the `-i` op
 
 ## Documentation
 To build and show the documentation, run
-````bash
+```bash
     $ make docs-sphinx
     $ make docs-show
-````
+```
 
 ## Contribute
 
 You are welcome to contribute to IOP4. Fork and create a PR!
+
+## Citing IOP4
+
+If you use IOP4, or any result derived with it, we kindly ask you to cite the following references:
+
+<div>
+<a href="https://zenodo.org/doi/10.5281/zenodo.10222722"><img src="https://zenodo.org/badge/636786270.svg" alt="DOI"></img></a>
+</div>
+
+You can use the following BibTeX entry:
+
+```bibtex
+@software{juan_escudero_2023_10222723,
+  author       = {Juan Escudero and
+                  Daniel Morcuende},
+  title        = {juanep97/iop4: v0.1.0},
+  month        = nov,
+  year         = 2023,
+  publisher    = {Zenodo},
+  version      = {v0.1.0},
+  doi          = {10.5281/zenodo.10222723},
+  url          = {https://doi.org/10.5281/zenodo.10222723}
+}
+```
+
+This might change in the future, as IOP4 is still under the process of being published in a peer-reviewed journal. Check this repository for updates :)

docs/citations.bib

@@ -34,7 +34,7 @@ @ARTICLE{patat:2006
       adsnote = {Provided by the SAO/NASA Astrophysics Data System}
 }
 
-@article{dipol:2020,
+@article{dipol:Piirola:2020,
 	author = {{Piirola, V.} and {Berdyugin, A.} and {Frisch, P. C.} and {Kagitani, M.} and {Sakanoi, T.} and {Berdyugina, S.} and {Cole, A. A.} and {Harlingten, C.} and {Hill, K.}},
 	title = {High-precision polarimetry of nearby stars (d < 50 pc) - Mapping the interstellar dust and magnetic field inside the Local Bubble},
 	DOI= "10.1051/0004-6361/201937324",
@@ -45,7 +45,103 @@ @article{dipol:2020
 	pages = "A46",
 }
 
+@ARTICLE{dipol:Jorge:2024,
+       author = {{Otero-Santos}, J. and {Piirola}, V. and {Escudero Pedrosa}, J. and {Agudo}, I. and {Morcuende}, D. and {Sota}, A. and {Casanova}, V. and {Aceituno}, F.~J. and {Santos-Sanz}, P.},
+        title = "{Characterization of High-polarization Stars and Blazars with DIPOL-1 at Sierra Nevada Observatory}",
+      journal = {\aj},
+     keywords = {Polarimeters, Polarimetry, Blazars, Jets, High energy astrophysics, Observational astronomy, 1277, 1278, 164, 870, 739, 1145, Astrophysics - Instrumentation and Methods for Astrophysics, Astrophysics - High Energy Astrophysical Phenomena},
+         year = 2024,
+        month = mar,
+       volume = {167},
+       number = {3},
+          eid = {137},
+        pages = {137},
+          doi = {10.3847/1538-3881/ad250d},
+archivePrefix = {arXiv},
+       eprint = {2312.05312},
+ primaryClass = {astro-ph.IM},
+       adsurl = {https://ui.adsabs.harvard.edu/abs/2024AJ....167..137O},
+      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
+
 @online{astropy:ccd_data_reduction_guide,
       title        = {CCD Data Reduction Guide},
       url          = {https://www.astropy.org/ccd-reduction-and-photometry-guide/},
-}
+}
+
+@ARTICLE{Nilsson:2007,
+       author = {{Nilsson}, K. and {Pasanen}, M. and {Takalo}, L.~O. and {Lindfors}, E. and {Berdyugin}, A. and {Ciprini}, S. and {Pforr}, J.},
+        title = "{Host galaxy subtraction of TeV candidate BL Lacertae objects}",
+      journal = {\aap},
+     keywords = {galaxies: active, BL Lacertae objects: general, techniques: photometric, methods: data analysis, Astrophysics},
+         year = 2007,
+        month = nov,
+       volume = {475},
+       number = {1},
+        pages = {199-207},
+          doi = {10.1051/0004-6361:20077624},
+archivePrefix = {arXiv},
+       eprint = {0709.2533},
+ primaryClass = {astro-ph},
+       adsurl = {https://ui.adsabs.harvard.edu/abs/2007A&A...475..199N},
+      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
+
+@online{photutils,
+author       = {Larry Bradley and
+                Brigitta Sip{\H o}cz and
+                Thomas Robitaille and
+                Erik Tollerud and
+                Z\`e Vin{\'{\i}}cius and
+                Christoph Deil and
+                Kyle Barbary and
+                Tom J Wilson and
+                Ivo Busko and
+                Axel Donath and
+                Hans Moritz G{\"u}nther and
+                Mihai Cara and
+                P. L. Lim and
+                Sebastian Me{\ss}linger and
+                Simon Conseil and
+                Azalee Bostroem and
+                Michael Droettboom and
+                E. M. Bray and
+                Lars Andersen Bratholm and
+                Geert Barentsen and
+                Matt Craig and
+                Shivangee Rathi and
+                Sergio Pascual and
+                Gabriel Perren and
+                Iskren Y. Georgiev and
+                Miguel de Val-Borro and
+                Wolfgang Kerzendorf and
+                Yoonsoo P. Bach and
+                Bruno Quint and
+                Harrison Souchereau},
+title        = {astropy/photutils: 1.8.0},
+month        = may,
+year         = 2023,
+publisher    = {Zenodo},
+version      = {1.8.0},
+doi          = {10.5281/zenodo.7946442},
+url          = {https://doi.org/10.5281/zenodo.7946442}
+}
+
+@ARTICLE{lang:2010,
+       author = {{Lang}, Dustin and {Hogg}, David W. and {Mierle}, Keir and {Blanton}, Michael and {Roweis}, Sam},
+        title = "{Astrometry.net: Blind Astrometric Calibration of Arbitrary Astronomical Images}",
+      journal = {\aj},
+     keywords = {astrometry, catalogs, instrumentation: miscellaneous, methods: data analysis, methods: statistical, techniques: image processing, Astrophysics - Instrumentation and Methods for Astrophysics},
+         year = 2010,
+        month = may,
+       volume = {139},
+       number = {5},
+        pages = {1782-1800},
+          doi = {10.1088/0004-6256/139/5/1782},
+archivePrefix = {arXiv},
+       eprint = {0910.2233},
+ primaryClass = {astro-ph.IM},
+       adsurl = {https://ui.adsabs.harvard.edu/abs/2010AJ....139.1782L},
+      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
+

docs/conf.py

@@ -1,3 +1,12 @@
+import sys, os, subprocess
+GIT_COMMIT_HASH = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=os.path.dirname(os.path.realpath(__file__))).decode('ascii').strip()
+GIT_BRANCH = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=os.path.dirname(os.path.realpath(__file__))).decode('ascii').strip()
+GIT_DESCRIBE = subprocess.check_output(['git', 'describe', '--always'], cwd=os.path.dirname(os.path.realpath(__file__))).decode('ascii').strip()
+
+# thumnail_gallery extension
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent / "sphinxext"))
+
 # Configuration file for the Sphinx documentation builder.
 #
 # For the full list of built-in configuration values, see the documentation:
@@ -6,24 +15,50 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
+# Get exact version from git
+
 project = 'IOP4'
 copyright = '2023, Juan Escudero Pedrosa'
 author = 'Juan Escudero Pedrosa'
-release = '0.0.1'
+release = GIT_DESCRIBE
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
+    'myst_nb',
+    "nbsphinx",
+    'thumbnail_gallery',
     'sphinxcontrib.bibtex',
     'sphinx.ext.autodoc',
-    'sphinx_mdinclude',
     'sphinx.ext.linkcode',
     'numpydoc',
 ]
 
 templates_path = ['_templates']
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['conf.py', '_build', 'sphinxext', '.*']
+source_suffix = ['.rst', '.md']
+
+nb_custom_formats = {
+    '.py': ['jupytext.reads', {'fmt': 'py:percent'}],
+}
+
+nb_merge_streams = True # otherwise several prints stmts result in multiple chunks
+nb_execution_timeout = 60 # seconds
+
+# automatically extract thumbnails from notebooks (with sphinxext/thumbnail_gallery)
+
+def get_thumbnails():
+    from glob import glob
+    from pathlib import Path
+
+    nb_paths = glob(f"recipes/*.py")
+    nb_names = [Path(nb_path).stem for nb_path in nb_paths]
+    nbsphinx_thumbnails = {f"recipes/{nb_name}":f"_thumbnails/{nb_name}.png" for nb_name in nb_names}
+
+    return nbsphinx_thumbnails
+
+nbsphinx_thumbnails = get_thumbnails()
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
@@ -147,7 +182,7 @@ def linkcode_resolve(domain, info):
         print(f"basedir: {iop4conf.basedir}")
         rel_path = os.path.relpath(file_path, iop4conf.basedir)
         print(f"rel_path: {rel_path}")
-        url = f'{repo_url}/blob/main/{rel_path}'
+        url = f'{repo_url}/blob/{GIT_COMMIT_HASH}/{rel_path}'
         return url
     else:
         return None