diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml new file mode 100644 index 00000000..81c31c7a --- /dev/null +++ b/.github/workflows/deploy-image.yml @@ -0,0 +1,41 @@ +name: Create and publish a Docker image + +on: + push: + branches: ['master'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..afde6114 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +FROM continuumio/miniconda3 + +LABEL Description="ShapePipe Docker Image" +ENV SHELL /bin/bash + +ARG CC=gcc-9 +ARG CXX=g++-9 + +# gcc < 10 is required to compile ww +ENV CC=gcc-9 +ENV CXX=g++-9 + +RUN apt-get update --allow-releaseinfo-change && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install apt-utils -y && \ + apt-get install make -y && \ + apt-get install automake -y && \ + apt-get install autoconf -y && \ + apt-get install gcc-9 g++-9 -y && \ + apt-get install locales -y && \ + apt-get install libgl1-mesa-glx -y && \ + apt-get install xterm -y && \ + apt-get install cmake protobuf-compiler -y && \ + apt-get install libtool libtool-bin libtool-doc -y && \ + apt-get install libfftw3-bin libfftw3-dev -y && \ + apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \ + apt-get install vim -y && \ + apt-get install locate -y && \ + apt-get install curl -y && \ + apt-get install acl -y && \ + apt-get install sssd -y && \ + apt-get clean + +ADD nsswitch.conf /etc/ + +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +SHELL ["/bin/bash", "--login", "-c"] + +COPY ./environment.yml ./ +COPY install_shapepipe README.rst setup.py setup.cfg ./ +RUN touch ./README.md + +RUN conda update -n base -c defaults conda -c defaults +RUN conda env create --file environment.yml + +COPY shapepipe ./shapepipe +COPY scripts ./scripts + +# Make RUN commands use the new environment: +SHELL ["conda", "run", "-n", "shapepipe", "/bin/bash", "-c"] +RUN pip install jupyter diff --git a/environment.yml b/environment.yml index 9279402e..759813a8 100644 --- a/environment.yml +++ b/environment.yml @@ -4,30 +4,29 @@ channels: dependencies: - python=3.9 - pip>=21.2.4 + - astromatic-psfex==3.21.1 + - astromatic-source-extractor==2.25.0 + - astropy==5.1 + - automake + - autoconf + - cmake + - galsim + - joblib + - libtool + - matplotlib + - numba - numpy==1.21.6 - - astropy==5.0 - - automake==1.16.2 - - autoconf==2.69 - - cmake==3.19.6 - - galsim==2.2.5 - - joblib==1.1.0 - - libtool==2.4.6 - - matplotlib==3.5.1 - - numba==0.54.1 - - pandas==1.4.1 - pip: - - cs_util==0.0.5 - - mccd==1.2.3 - - modopt==1.6.0 - - PyQt5==5.15.6 - - pyqtgraph==0.12.4 - - python-pysap==0.0.6 - - reproject==0.8 - - sip_tpv==1.1 - - sf_tools==2.0.4 - - sqlitedict==2.0.0 - - termcolor==1.1.0 - - tqdm==4.63.0 - - treecorr==4.2.6 + - cs_util + - modopt + - PyQt5 + - pyqtgraph + - reproject + - sip_tpv + - sf_tools + - sqlitedict + - termcolor + - tqdm + - treecorr - git+https://github.com/aguinot/ngmix@stable_version - git+https://github.com/tobias-liaudat/Stile@v0.1 diff --git a/example/cfis/config_exp_SpMh.ini b/example/cfis/config_exp_SpMh.ini index 76f87ddd..20ac3861 100644 --- a/example/cfis/config_exp_SpMh.ini +++ b/example/cfis/config_exp_SpMh.ini @@ -63,7 +63,7 @@ FILE_EXT = .fitsfz, .fitsfz, .fitsfz NUMBERING_SCHEME = -0000000 # OUTPUT_SUFFIX, actually file name prefixes. -# Expected keyword "flag" will lead to a behavior where the data are save as int. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. # The code also expects the image data to use the "image" suffix # (default value in the pipeline). OUTPUT_SUFFIX = image, weight, flag diff --git a/install_shapepipe b/install_shapepipe index c370d91d..96cc8939 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -25,20 +25,6 @@ libpng_ver="1.6.37" mpi4py_ver="3.1.3" openblas_ver="0.3.18" -# SExtractor Package -sex_url="https://github.com/astromatic/sextractor/archive/2.25.0.tar.gz" -sex_tar="2.25.0.tar.gz" -sex_dir="sextractor-2.25.0" -sex_bin="sex" -sex_version=TRUE - -# PSFEx Package -psfex_url="https://github.com/astromatic/psfex/archive/3.21.1.tar.gz" -psfex_tar="3.21.1.tar.gz" -psfex_dir="psfex-3.21.1" -psfex_bin="psfex" -psfex_version=TRUE - # WeightWatcher Package ww_url="http://snapshot.debian.org/archive/debian/20171211T160522Z/pool/main/w/weightwatcher/weightwatcher_1.12.orig.tar.gz" ww_tar="weightwatcher_1.12.orig.tar.gz" @@ -84,8 +70,6 @@ ENV_NAME='shapepipe' BUILD_ENV=TRUE DEVELOP=FALSE VOS=FALSE -INSTALL_SEX=TRUE -INSTALL_PSFEX=TRUE INSTALL_WW=TRUE INSTALL_CDSCLIENT=TRUE ATLAS_SEARCH=FALSE @@ -105,66 +89,6 @@ download_package() { rm $2 } -# Function to build SExtractor with OpenBLAS -build_sex_blas() { - astromatic_setup - cd $BUILD_DIR/$1 - ./autogen.sh - ./configure --prefix=$CONDA_PREFIX --enable-openblas --with-fftw-libdir=$FFTW_LIB \ - --with-fftw-incdir=$FFTW_INC --with-openblas-libdir=$BLAS_LIB \ - --with-openblas-incdir=$BLAS_INC --quiet - make -j --quiet - make install -} - -# Function to build SExtractor with ATLAS -build_sex_atlas() { - astromatic_setup - cd $BUILD_DIR/$1 - ./autogen.sh - if [ "$ATLAS_SEARCH" == TRUE ] - then - ./configure --prefix=$CONDA_PREFIX --with-fftw-libdir=$FFTW_LIB \ - --with-fftw-incdir=$FFTW_INC --quiet - else - ./configure --prefix=$CONDA_PREFIX --with-fftw-libdir=$FFTW_LIB \ - --with-fftw-incdir=$FFTW_INC --with-atlas-libdir=$ATLAS_LIB \ - --with-atlas-incdir=$ATLAS_INC --quiet - fi - make -j --quiet - make install -} - -# Function to build PSFEx with OpenBLAS -build_psfex_blas() { - astromatic_setup - cd $BUILD_DIR/$1 - ./autogen.sh - ./configure --prefix=$CONDA_PREFIX --enable-openblas --enable-plplot=no \ - --with-fftw-libdir=$FFTW_LIB --with-fftw-incdir=$FFTW_INC \ - --with-openblas-libdir=$BLAS_LIB --with-openblas-incdir=$BLAS_INC --quiet - make -j --quiet - make install -} - -# Function to build PSFEx with ATLAS -build_psfex_atlas() { - astromatic_setup - cd $BUILD_DIR/$1 - ./autogen.sh - if [ "$ATLAS_SEARCH" == TRUE ] - then - ./configure --prefix=$CONDA_PREFIX --enable-plplot=no \ - --with-fftw-libdir=$FFTW_LIB --with-fftw-incdir=$FFTW_INC --quiet - else - ./configure --prefix=$CONDA_PREFIX --enable-plplot=no \ - --with-fftw-libdir=$FFTW_LIB --with-fftw-incdir=$FFTW_INC \ - --with-atlas-libdir=$ATLAS_LIB --with-atlas-incdir=$ATLAS_INC --quiet - fi - make -j --quiet - make install -} - # Function to build a standard package build_standard() { cd $BUILD_DIR/$1 @@ -290,8 +214,6 @@ start() { # Function to report package versions packages() { - check_binary2 $sex_bin "SExtractor" $sex_version $INSTALL_SEX - check_binary2 $psfex_bin "PSFEx" $psfex_version $INSTALL_PSFEX check_binary2 $ww_bin "WeightWatcher" $ww_version $INSTALL_WW check_binary2 $cdsclient_bin "CDSclient" $cdsclient_version $INSTALL_CDSCLIENT check_binary2 $mpi_bin "MPI" $mpi_version $USE_MPI $MPI_ROOT @@ -332,8 +254,6 @@ package_report() { then activate_shapepipe fi - INSTALL_SEX=FALSE - INSTALL_PSFEX=FALSE INSTALL_WW=FALSE INSTALL_CDSCLIENT=FALSE USE_MPI=FALSE @@ -381,8 +301,6 @@ setup() { check_status 'Operating System:' $SYSOS check_status 'Conda Version:' $CONDA_VERSION check_status 'Build Conda Environment:' $BUILD_ENV $ENV_NAME - check_status 'Install SExtractor:' $INSTALL_SEX - check_status 'Install PSFEx:' $INSTALL_PSFEX check_status 'Install WeightWatcher:' $INSTALL_WW check_status 'Install CDSclient:' $INSTALL_CDSCLIENT check_status 'Use MPI:' $USE_MPI $MPI_ROOT @@ -467,8 +385,6 @@ Options:\n \t--develop\t option to install additional development packages\n \t--vos\t\t option to install virtual observatory software\n \t--no-env\t do not build Conda environment\n -\t--no-sex\t do not build SExtractor\n -\t--no-psfex\t do not build PSFEx\n \t--no-ww\t\t do not build WeightWatcher\n \t--no-cds\t do not build CDSclient\n \t--no-exe\t do not build any system executables\n\n @@ -537,14 +453,6 @@ case $i in BUILD_ENV=FALSE shift ;; - --no-sex) - INSTALL_SEX=FALSE - shift - ;; - --no-psfex) - INSTALL_PSFEX=FALSE - shift - ;; --no-ww) INSTALL_WW=FALSE shift @@ -590,8 +498,6 @@ case $i in shift ;; --no-exe) - INSTALL_SEX=FALSE - INSTALL_PSFEX=FALSE INSTALL_WW=FALSE INSTALL_CDSCLIENT=FALSE shift @@ -716,37 +622,6 @@ fi # BUILD EXTERNAL EXECUTABLES ############################################################################## -# Build SExtractor -if [ "$INSTALL_SEX" == TRUE ] && check_binary $sex_bin -then - report_progress 'SExtractor' - conda install -n $ENV_NAME -c conda-forge "fftw==${fftw_ver}" -y - download_package $sex_url $sex_tar - if [ "$use_atlas" == TRUE ] - then - build_sex_atlas $sex_dir - else - conda install -n $ENV_NAME -c conda-forge "openblas==${openblas_ver}" -y - build_sex_blas $sex_dir - fi -fi - -# Build PSFEx -if [ "$INSTALL_PSFEX" == TRUE ] && check_binary $psfex_bin -then - report_progress 'PSFEx' - conda install "libpng==${libpng_ver}" -y - conda install -n $ENV_NAME -c conda-forge "fftw==${fftw_ver}" -y - download_package $psfex_url $psfex_tar - if [ "$use_atlas" == TRUE ] - then - build_psfex_atlas $psfex_dir - else - conda install -n $ENV_NAME -c conda-forge "openblas==${openblas_ver}" -y - build_psfex_blas $psfex_dir - fi -fi - # Build WeightWatcher if [ "$INSTALL_WW" == TRUE ] && check_binary $ww_bin then diff --git a/nsswitch.conf b/nsswitch.conf new file mode 100644 index 00000000..22d8d994 --- /dev/null +++ b/nsswitch.conf @@ -0,0 +1,62 @@ +# +# /etc/nsswitch.conf +# +# An example Name Service Switch config file. This file should be +# sorted with the most-used services at the beginning. +# +# The entry '[NOTFOUND=return]' means that the search for an +# entry should stop if the search in the previous entry turned +# up nothing. Note that if the search failed due to some other reason +# (like no NIS server responding) then the search continues with the +# next entry. +# +# Valid entries include: +# +# nisplus Use NIS+ (NIS version 3) +# nis Use NIS (NIS version 2), also called YP +# dns Use DNS (Domain Name Service) +# files Use the local files +# db Use the local database (.db) files +# compat Use NIS on compat mode +# hesiod Use Hesiod for user lookups +# [NOTFOUND=return] Stop searching if not found so far +# + +# To use db, put the "db" in front of "files" for entries you want to be +# looked up first in the databases +# +# Example: +#passwd: db files nisplus nis +#shadow: db files nisplus nis +#group: db files nisplus nis + +passwd: sss files +shadow: files sss +group: sss files + +#hosts: db files nisplus nis dns +hosts: files dns + +# Example - obey only what nisplus tells us... +#services: nisplus [NOTFOUND=return] files +#networks: nisplus [NOTFOUND=return] files +#protocols: nisplus [NOTFOUND=return] files +#rpc: nisplus [NOTFOUND=return] files +#ethers: nisplus [NOTFOUND=return] files +#netmasks: nisplus [NOTFOUND=return] files + +bootparams: nisplus [NOTFOUND=return] files + +ethers: files +netmasks: files +networks: files +protocols: files +rpc: files +services: files + +netgroup: nisplus + +publickey: nisplus + +automount: files nisplus +aliases: files nisplus diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py index 1a160c44..076f5c93 100755 --- a/scripts/python/merge_final_cat.py +++ b/scripts/python/merge_final_cat.py @@ -20,8 +20,6 @@ from optparse import OptionParser -from tqdm import tqdm - from shapepipe.utilities import cfis diff --git a/scripts/sh/curl_canfar.sh b/scripts/sh/curl_canfar.sh new file mode 100755 index 00000000..89bb4f64 --- /dev/null +++ b/scripts/sh/curl_canfar.sh @@ -0,0 +1,44 @@ +#!/usr/bin/bash + +# Usage +# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.7 shapepipe/scripts/sh/init_canfar.sh 000.000 + +SSL=~/.ssl/cadcproxy.pem +NCORE=16 +SESSION=https://ws-uv.canfar.net/skaha/v0/session +RESOURCES="ram=16&cores=$NCORE" +IMAGE=images.canfar.net/unions/shapepipe +NAME=shapepipe + +# version of image on canfar, e.g. 0:7, 0:8 +version=$1 + +# command on canfar, e.g. shapepipe/scripts/sh/init_run_canfar.sh +cmd=$2 + +# command line argument, e.g. 181.308 +arg="$3 $NCORE" + +#arg_an=`echo "$arg" | tr '_' 'X' | tr '.' 'X'` +arg_an=X + +echo +echo "Start headless container" +echo "========================" +ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}${arg_an}" -d "cmd=$cmd" --data-urlencode "args=$arg"` +echo $ID + +echo +echo "Events (incl. errors)" +echo "=====================" +cmd="curl -E $SSL $SESSION/$ID?view=events" +echo $cmd +$cmd + +echo +echo "Logs (incl. stdout)" +echo "===================" +cmd="curl -E $SSL $SESSION/$ID?view=logs" +echo $cmd +$cmd + diff --git a/scripts/sh/init_canfar.sh b/scripts/sh/init_canfar.sh new file mode 100755 index 00000000..dd3346f9 --- /dev/null +++ b/scripts/sh/init_canfar.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo "start init canfar" + +echo init_canfar > ~/init_canfar.log +date >> ~/init_canfar.log + +. /opt/conda/etc/profile.d/conda.sh + +conda activate shapepipe + +echo "end init canfar" + diff --git a/shapepipe/modules/python_example_runner.py b/shapepipe/modules/python_example_runner.py index d468c10e..fc5565ec 100644 --- a/shapepipe/modules/python_example_runner.py +++ b/shapepipe/modules/python_example_runner.py @@ -19,10 +19,8 @@ 'astropy', 'galsim', 'joblib', - 'mccd', 'ngmix', 'pandas', - 'pysap', 'scipy', 'sf_tools', 'sip_tpv', diff --git a/xterm-start.sh b/xterm-start.sh new file mode 100755 index 00000000..007a2516 --- /dev/null +++ b/xterm-start.sh @@ -0,0 +1,2 @@ +#!/bin/bash +xterm