Netcdf46 (#249)

* Fix python3 slice issue(setitem) and flake8 * update to libnetcdf 4.6 * try circleci unstable label * try version 2 circleci * try version 2 circleci * change workflow name * change cdtime to cdms * add certificate to circleci * add fix conda-upload in circleci 2.0 * update prep_for_build version * fix curl command * fix cicleci for cdms * use unstable channel change uvcdat for cdat * build cdms on circleci 2.0 * fix circleci config.yml * change Users/distiler to /Users/denisnadeau * add gcc_linux * add LDSHARED for linux * disable cert and py results * add gcc_linux-64 * change cdscan link * fix myproxy * add esmf and esmpy to py3 env
CDAT · Jun 20, 2018 · e1b587f · e1b587f
1 parent 405fe24
commit e1b587f
Show file tree

Hide file tree

Showing 9 changed files with 171 additions and 232 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,147 @@
+version: 2
+
+checkout:
+  post:
+    - ./ci-support/checkout_merge_commit.sh
+
+aliases:
+
+  - &setup_miniconda
+    name: setup_miniconda
+    command: |
+      mkdir -p workspace
+      git clone -b validateNightly [email protected]:CDAT/cdat workspace/cdat
+      ls workspace/cdat
+      # following will install miniconda3 under $WORKDIR/miniconda/bin
+      python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
+      
+  - &create_conda_env
+    name: create_conda_env
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       conda config --set always_yes yes --set changeps1 no
+       conda update -y -q conda
+       conda config --set anaconda_upload no
+       conda create -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient "python>3"
+       conda create -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 "python<3"
+       if [ $(uname) == "Linux" ]; then
+           conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64
+           conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64
+       else
+           conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc
+           conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc
+       fi
+
+  - &setup_cdms
+    name: setup_cdms
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       export UVCDAT_ANONYMOUS_LOG=False
+       source activate py3
+       mkdir $HOME/.esg
+       echo "Get ESGF certificates"
+       echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o $HOME/.esg/esgf.cert
+       cp tests/dodsrccircleci $HOME/.dodsrc
+       echo "Create .dods_cookies"
+       curl -L  -v   -c $HOME/.esg/.dods_cookies --cert $HOME/.esg/esgf.cert  --key $HOME/.esg/esgf.cert "https://aims3.llnl.gov/thredds/dodsC/cmip5_css02_data/cmip5/output1/CMCC/CMCC-CM/decadal2005/mon/atmos/Amon/r1i1p1/cct/1/cct_Amon_CMCC-CM_decadal2005_r1i1p1_202601-203512.nc.dds"
+       if [ $(uname) == "Linux" ];then
+         export LDSHARED="$CC -shared -pthread"
+         LDSHARED="$CC -shared -pthread" python setup.py install
+       else
+         python setup.py install
+       fi
+       source activate py2
+       rm -rf build
+       if [ $(uname) == "Linux" ];then
+         export LDSHARED="$CC -shared -pthread"
+         LDSHARED="$CC -shared -pthread" python setup.py install
+       else
+         python setup.py install
+       fi
+
+  - &run_cdms_tests
+    name: run_cdms_tests
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       export UVCDAT_ANONYMOUS_LOG=False
+       set -e
+       source activate py2
+       python run_tests.py -v2 
+       PY2_RESULT=$?
+       echo "*** py2 test result: "${PY2_RESULT}
+       source activate py3
+       python run_tests.py -v2 
+       PY3_RESULT=$?
+       echo "*** py3 test result: "${PY3_RESULT}
+       echo  $PY2_RESULT > $HOME/project/$WORKDIR/py2_result.txt
+       echo  $PY3_RESULT > $HOME/project/$WORKDIR/py3_result.txt
+
+  - &upload_cdms
+    name: upload_cdms
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       export LABEL="nightly";
+       # Retrieve results
+       PY2_RESULT=$(cat $HOME/project/$WORKDIR/py2_result.txt)
+       PY3_RESULT=$(cat $HOME/project/$WORKDIR/py3_result.txt)
+       echo "*** CIRCLE BRANCH: "${CIRCLE_BRANCH}
+       echo "*** PY2_RESULT: "${PY2_RESULT}
+       echo "*** PY3_RESULT: "${PY3_RESULT}
+       if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then bash ./ci-support/conda_upload.sh ; fi
+       if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then conda install -n root conda-build anaconda-client ; fi
+       #if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH != "master" ]; then
+       if [ $CIRCLE_BRANCH != "master" ]; then
+           export LABEL="unstable";
+           echo "NOTE: upload LABEL="${LABEL}
+           conda install -n root conda-build anaconda-client;
+           bash ./ci-support/conda_upload.sh;
+       fi
+
+jobs:
+  macos_cdms:
+    macos:
+      xcode: "9.2.0"
+    environment:
+      WORKDIR: "workspace/test_macos_cdms"
+    steps:
+      - checkout
+      - run: *setup_miniconda
+      - run: *create_conda_env
+      - run: *setup_cdms
+      - run: *run_cdms_tests
+      - run: *upload_cdms
+      - store_artifacts:
+          path: tests_html
+          destination: tests_html
+      - store_artifacts:
+          path: tests_png
+          destination: tests_png
+
+  linux_cdms:
+    machine:
+      image: circleci/classic:latest
+    environment:
+      WORKDIR: "workspace/test_linux_cdms"
+    steps:
+      - checkout
+      - run: *setup_miniconda
+      - run: *create_conda_env
+      - run: *setup_cdms
+      - run: *run_cdms_tests
+      - run: *upload_cdms
+      - store_artifacts:
+          path: tests_html
+          destination: tests_html
+      - store_artifacts:
+          path: tests_png
+          destination: tests_png
+
+
+workflows:
+  version: 2
+  cdms_test:
+    jobs:
+      - macos_cdms
+      - linux_cdms
+
+
diff --git a/.travis.yml b/.travis.yml
@@ -20,7 +20,7 @@ before_install:
 script:
 #- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
 - conda create -n py3 python=3.6
-- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient 
+- conda install -n py3 -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient 
 - source activate py3
 - if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy ; fi
 - export UVCDAT_ANONYMOUS_LOG=False
@@ -37,7 +37,7 @@ script:
 - conda create -n py2 python=2.7
 - source activate py2
 - which python
-- conda install -c nesii/channel/dev-esmf -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
+- conda install -c nesii/channel/dev-esmf -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
 - export UVCDAT_ANONYMOUS_LOG=False
 - python setup.py install
 - python run_tests.py -v2 -s

diff --git a/ci-support/circleci_mac.sh b/ci-support/circleci_mac.sh
diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh
diff --git a/ci-support/circleci_mac_machine_pre.sh b/ci-support/circleci_mac_machine_pre.sh
diff --git a/ci-support/conda_upload.sh b/ci-support/conda_upload.sh
@@ -1,83 +1,34 @@
 #!/usr/bin/env bash
 PKG_NAME=cdms2
 USER=cdat
-echo "Trying to upload conda"
-mkdir ${HOME}/conda-bld
-export CONDA_BLD_PATH=${HOME}/conda-bld
-export VERSION="2.12"
+export VERSION="3.0"
+echo "Trying to upload to conda"
+echo ""
+echo "Activating base env"
+source activate base
+echo "Making sure conda-build is installed"
+conda install "conda-build<3.10"
+echo "Updating conda"
+conda update -y -q conda
 if [ `uname` == "Linux" ]; then
     OS=linux-64
     echo "Linux OS"
-    yum install -y wget git gcc
-    # wget --no-check https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh  -O miniconda3.sh 2> /dev/null
-    wget --no-check https://repo.continuum.io/miniconda/Miniconda2-4.3.30-Linux-x86_64.sh  -O miniconda2.sh 2> /dev/null
-    bash miniconda2.sh -b -p ${HOME}/miniconda
-    export SYSPATH=$PATH
-    export PATH=${HOME}/miniconda/bin:${SYSPATH}
-    echo $PATH
-    conda config --set always_yes yes --set changeps1 no
-    conda config --set anaconda_upload false --set ssl_verify false
-    conda install -n root -q anaconda-client "conda-build<3.3"
-    conda install -n root gcc future
-    which python
-    export UVCDAT_ANONYMOUS_LOG=False
-    BRANCH=${TRAVIS_BRANCH}
-#    echo "Creating python 3 env"
-#    conda create -n py3 python=3.6
-#    conda install -n py3 -c conda-forge -c uvcdat setuptools libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient numpy
-#    conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy
-#    echo "Creating certificate"
-#    source activate py3
-#    mkdir ${HOME}/.esg
-#    echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o ${HOME}/.esg/esgf.cert
-#    ls ${HOME}/.esg
-#    cd travis_home
-#    ls
-#    cp tests/dodsrc ${HOME}.dodsrc
-#    source deactivate
 else
     echo "Mac OS"
     OS=osx-64
-    BRANCH=${CIRCLE_BRANCH}
 fi
 
-which python
-if [ `uname` == "Linux" ]; then
-    conda install -n root -q anaconda-client "conda-build<3.3"
-else
-    conda install -n root -q anaconda-client conda-build
-fi
-# pin conda so that conda-build does not update it
-#if [ `uname` == "Darwin" ]; then
-#    echo "conda ==4.3.21" >> ~/miniconda/conda-meta/pinned  # Pin conda as workaround for conda/conda#6030
-#fi
+mkdir ~/conda-bld
 conda config --set anaconda_upload no
+export CONDA_BLD_PATH=${HOME}/conda-bld
 echo "Cloning recipes"
-cd ${HOME}
-git clone git://github.com/UV-CDAT/conda-recipes
+git clone git://github.com/CDAT/conda-recipes
 cd conda-recipes
 # uvcdat creates issues for build -c uvcdat confises package and channel
 rm -rf uvcdat
-python ./prep_for_build.py -b ${BRANCH}
-echo "Building now"
-echo "use nesii/label/dev-esmf for esmf"
-conda build -V
-conda build $PKG_NAME -c nesii/label/dev-esmf  -c uvcdat/label/nightly -c conda-forge -c uvcdat 
-#
-# binstar config set 'false' instead of false (not quote) I have to do it manually
-# this is true for OSX.
-# binstar is changing verify_ssl to ssl_verify, but the later is not always working
-# 
-# binstar config --set verify_ssl false
-# binstar config --set ssl_verify false
-#
-mkdir -p ~/.continuum/anaconda-client/
-echo "ssl_verify: false" >> ~/.continuum/anaconda-client/config.yaml
-echo "verify_ssl: false" >> ~/.continuum/anaconda-client/config.yaml
-if [ `uname` == "Darwin" ]; then
-    # fix conda and anaconda-client conflict
-    conda install conda==4.2.16
-fi
-anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l ${LABEL}  ${CONDA_BLD_PATH}/$OS/$PKG_NAME-$VERSION.`date +%Y`*_0.tar.bz2 --force
-
+export BRANCH=${CIRCLE_BRANCH}
+python ./prep_for_build.py  -b ${BRANCH}
 
+conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge  --python 3.6
+conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge  --python 2.7
+anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL $CONDA_BLD_PATH/$OS/${PKG_NAME}-$VERSION.`date +%Y*`0.tar.bz2 --force