From 9d5b7d21c2566a5c5142d0f3d3a39f9690df75a0 Mon Sep 17 00:00:00 2001
From: Denis Nadeau <dnadeau4@users.noreply.github.com>
Date: Tue, 5 Jun 2018 18:14:24 -0700
Subject: [PATCH] Netcdf46 (#249)

* Fix python3 slice issue(setitem) and flake8

* update to libnetcdf 4.6

* try circleci unstable label

* try version 2 circleci

* try version 2 circleci

* change workflow name

* change cdtime to cdms

* add certificate to circleci

* add fix conda-upload in circleci 2.0

* update prep_for_build version

* fix curl command

* fix cicleci for cdms

* use unstable channel change uvcdat for cdat

* build cdms on circleci 2.0

* fix circleci config.yml

* change Users/distiler to /Users/denisnadeau

* add gcc_linux

* add LDSHARED for linux

* disable cert and py results

* add gcc_linux-64

* change cdscan link

* fix myproxy

* add esmf and esmpy to py3 env
---
 .circleci/config.yml                   | 147 +++++++++++++++++++++++++
 .travis.yml                            |   4 +-
 ci-support/circleci_mac.sh             |  15 ---
 ci-support/circleci_mac_dep.sh         | 102 -----------------
 ci-support/circleci_mac_machine_pre.sh |   8 --
 ci-support/conda_upload.sh             |  83 +++-----------
 circle.yml                             |  35 ------
 tests/dodsrccircleci                   |   8 +-
 tests/test_cdscan.py                   |   1 +
 9 files changed, 171 insertions(+), 232 deletions(-)
 create mode 100644 .circleci/config.yml
 delete mode 100644 ci-support/circleci_mac.sh
 delete mode 100644 ci-support/circleci_mac_dep.sh
 delete mode 100644 ci-support/circleci_mac_machine_pre.sh
 delete mode 100644 circle.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 00000000..8a582e3b
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,147 @@
+version: 2
+
+checkout:
+  post:
+    - ./ci-support/checkout_merge_commit.sh
+        
+aliases:
+
+  - &setup_miniconda
+    name: setup_miniconda
+    command: |
+      mkdir -p workspace
+      git clone -b validateNightly git@github.com:CDAT/cdat workspace/cdat
+      ls workspace/cdat
+      # following will install miniconda3 under $WORKDIR/miniconda/bin
+      python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
+      
+  - &create_conda_env
+    name: create_conda_env
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       conda config --set always_yes yes --set changeps1 no
+       conda update -y -q conda
+       conda config --set anaconda_upload no
+       conda create -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient "python>3"
+       conda create -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 "python<3"
+       if [ $(uname) == "Linux" ]; then
+           conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64
+           conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64
+       else
+           conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc
+           conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc
+       fi
+
+  - &setup_cdms
+    name: setup_cdms
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       export UVCDAT_ANONYMOUS_LOG=False
+       source activate py3
+       mkdir $HOME/.esg
+       echo "Get ESGF certificates"
+       echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o $HOME/.esg/esgf.cert
+       cp tests/dodsrccircleci $HOME/.dodsrc
+       echo "Create .dods_cookies"
+       curl -L  -v   -c $HOME/.esg/.dods_cookies --cert $HOME/.esg/esgf.cert  --key $HOME/.esg/esgf.cert "https://aims3.llnl.gov/thredds/dodsC/cmip5_css02_data/cmip5/output1/CMCC/CMCC-CM/decadal2005/mon/atmos/Amon/r1i1p1/cct/1/cct_Amon_CMCC-CM_decadal2005_r1i1p1_202601-203512.nc.dds"
+       if [ $(uname) == "Linux" ];then
+         export LDSHARED="$CC -shared -pthread"
+         LDSHARED="$CC -shared -pthread" python setup.py install
+       else
+         python setup.py install
+       fi
+       source activate py2
+       rm -rf build
+       if [ $(uname) == "Linux" ];then
+         export LDSHARED="$CC -shared -pthread"
+         LDSHARED="$CC -shared -pthread" python setup.py install
+       else
+         python setup.py install
+       fi
+
+  - &run_cdms_tests
+    name: run_cdms_tests
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       export UVCDAT_ANONYMOUS_LOG=False
+       set -e
+       source activate py2
+       python run_tests.py -v2 
+       PY2_RESULT=$?
+       echo "*** py2 test result: "${PY2_RESULT}
+       source activate py3
+       python run_tests.py -v2 
+       PY3_RESULT=$?
+       echo "*** py3 test result: "${PY3_RESULT}
+       echo  $PY2_RESULT > $HOME/project/$WORKDIR/py2_result.txt
+       echo  $PY3_RESULT > $HOME/project/$WORKDIR/py3_result.txt
+
+  - &upload_cdms
+    name: upload_cdms
+    command: |
+       export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
+       export LABEL="nightly";
+       # Retrieve results
+       PY2_RESULT=$(cat $HOME/project/$WORKDIR/py2_result.txt)
+       PY3_RESULT=$(cat $HOME/project/$WORKDIR/py3_result.txt)
+       echo "*** CIRCLE BRANCH: "${CIRCLE_BRANCH}
+       echo "*** PY2_RESULT: "${PY2_RESULT}
+       echo "*** PY3_RESULT: "${PY3_RESULT}
+       if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then bash ./ci-support/conda_upload.sh ; fi
+       if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then conda install -n root conda-build anaconda-client ; fi
+       #if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH != "master" ]; then
+       if [ $CIRCLE_BRANCH != "master" ]; then
+           export LABEL="unstable";
+           echo "NOTE: upload LABEL="${LABEL}
+           conda install -n root conda-build anaconda-client;
+           bash ./ci-support/conda_upload.sh;
+       fi
+
+jobs:
+  macos_cdms:
+    macos:
+      xcode: "9.2.0"
+    environment:
+      WORKDIR: "workspace/test_macos_cdms"
+    steps:
+      - checkout
+      - run: *setup_miniconda
+      - run: *create_conda_env
+      - run: *setup_cdms
+      - run: *run_cdms_tests
+      - run: *upload_cdms
+      - store_artifacts:
+          path: tests_html
+          destination: tests_html
+      - store_artifacts:
+          path: tests_png
+          destination: tests_png
+
+  linux_cdms:
+    machine:
+      image: circleci/classic:latest
+    environment:
+      WORKDIR: "workspace/test_linux_cdms"
+    steps:
+      - checkout
+      - run: *setup_miniconda
+      - run: *create_conda_env
+      - run: *setup_cdms
+      - run: *run_cdms_tests
+      - run: *upload_cdms
+      - store_artifacts:
+          path: tests_html
+          destination: tests_html
+      - store_artifacts:
+          path: tests_png
+          destination: tests_png
+
+
+workflows:
+  version: 2
+  cdms_test:
+    jobs:
+      - macos_cdms
+      - linux_cdms
+
+       
diff --git a/.travis.yml b/.travis.yml
index 26d87cc8..0bb331b0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ before_install:
 script:
 #- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
 - conda create -n py3 python=3.6
-- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient 
+- conda install -n py3 -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient 
 - source activate py3
 - if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy ; fi
 - export UVCDAT_ANONYMOUS_LOG=False
@@ -37,7 +37,7 @@ script:
 - conda create -n py2 python=2.7
 - source activate py2
 - which python
-- conda install -c nesii/channel/dev-esmf -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
+- conda install -c nesii/channel/dev-esmf -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
 - export UVCDAT_ANONYMOUS_LOG=False
 - python setup.py install
 - python run_tests.py -v2 -s
diff --git a/ci-support/circleci_mac.sh b/ci-support/circleci_mac.sh
deleted file mode 100644
index 66e14816..00000000
--- a/ci-support/circleci_mac.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-export UVCDAT_ANONYMOUS_LOG=False
-export PATH=${HOME}/miniconda/bin:${PATH}
-echo "CIRCLE CI BRANCH:"$CIRCLE_BRANCH
-echo "CI_PULL_REQUESTS"$CI_PULL_REQUESTS
-echo "CI_PULL_REQUEST"$CI_PULL_REQUEST
-source activate py2
-python run_tests.py -v2 -s
-RESULT=$?
-source activate py3
-python run_tests.py -v2 -s
-RESULT=$(( $RESULT + $? ))
-echo "RESULT:"${RESULT}
-if [ $RESULT -eq 0 -a $CIRCLE_BRANCH != "master" ]; then LABEL='unstable' bash ./ci-support/conda_upload.sh ; fi
-if [ $RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then LABEL='nightly' bash ./ci-support/conda_upload.sh ; fi
-exit $RESULT
diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh
deleted file mode 100644
index 385aef61..00000000
--- a/ci-support/circleci_mac_dep.sh
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/usr/bin/env bash
-cmd="ls"
-echo $cmd
-$cmd
-
-cmd="pwd"
-echo $cmd
-$cmd
-
-cmd="export PATH=${HOME}/miniconda/bin:${PATH}"
-echo $cmd
-$cmd
-
-# Create Python 3 environment
-cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=5"
-echo $cmd
-$cmd
-
-# update openssl for myproxyclient
-cmd="conda install -n py3 pyopenssl"
-echo $cmd
-$cmd
-
-
-cmd="conda install -n py3 -c nesii/label/dev-esmf -c conda-forge esmf esmpy netcdf-fortran=4.4.4=5"
-echo $cmd
-$cmd
-
-# Create Python 2 environment
-cmd="conda create -n py2 python=2.7"
-echo $cmd
-$cmd
-
-# Activate python 2 environment
-cmd="source activate py2"
-echo $cmd
-$cmd 
-
-cmd="conda install -n py2 -c nesii/label/dev-esmf -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=5"
-echo $cmd
-$cmd
-
-# update openssl for myproxyclient
-cmd="conda install -n py2 pyopenssl"
-echo $cmd
-$cmd
-:
-
-# add relative path to ncdump
-#cmd="install_name_tool -change /usr/lib/libcurl.4.dylib @rpath/libcurl.4.dylib ${HOME}/miniconda/envs/py2/bin/ncdump"
-#echo $cmd
-#$cmd 
-
-#cmd="install_name_tool -change /usr/lib/libcurl.4.dylib @rpath/libcurl.4.dylib ${HOME}/miniconda/envs/py3/bin/ncdump"
-#echo $cmd
-#$cmd 
-
-cmd="export UVCDAT_ANONYMOUS_LOG=False"
-echo $cmd
-$cmd
-
-# Retrieve certificates from ESGF
-cmd="mkdir /Users/distiller/.esg"
-echo $cmd
-$cmd
-
-cmd="echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o /Users/distiller/.esg/esgf.cert "
-eval $cmd
-
-echo "Create .dods_cookies"
-cmd="curl -L  -v   -c /Users/distiller/.esg/.dods_cookies --cert /Users/distiller/.esg/esgf.cert  --key /Users/distiller/.esg/esgf.cert https://aims3.llnl.gov/thredds/dodsC/cmip5_css02_data/cmip5/output1/CMCC/CMCC-CM/decadal2005/mon/atmos/Amon/r1i1p1/cct/1/cct_Amon_CMCC-CM_decadal2005_r1i1p1_202601-203512.nc.dds"
-echo $cmd
-$cmd
-
-#cmd="openssl pkcs12 -export -inkey /Users/distiller/.esg/esgf.cert -in /Users/distiller/.esg/esgf.cert -name esgf  -out /Users/distiller/.esg/esgf.p12 -passout pass:esgf"
-#$cmd
-
-#cmd="sudo security import /Users/distiller/.esg/esgf.p12 -A -P esgf -k /Library/Keychains/System.keychain"
-#echo $cmd
-#$cmd
-
-#cmd="sudo security add-trusted-cert -d -r trustRoot -k '/Library/Keychains/System.keychain' /Users/distiller/.esg/esgf.cert" 
-#echo $cmd
-#$cmd
-
-#cmd="cp tests/dodsrccircleci /Users/distiller/.dodsrc"
-#echo $cmd
-#$cmd
-
-# compile cdms on py2 and py3 environemt.
-
-cmd="python setup.py install"
-echo $cmd
-$cmd
-
-cmd="source activate py3"
-echo $cmd
-$cmd 
-
-cmd="python setup.py install"
-echo $cmd
-$cmd
diff --git a/ci-support/circleci_mac_machine_pre.sh b/ci-support/circleci_mac_machine_pre.sh
deleted file mode 100644
index 386629ec..00000000
--- a/ci-support/circleci_mac_machine_pre.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-curl https://repo.continuum.io/miniconda/Miniconda3-4.3.30.1-MacOSX-x86_64.sh -o miniconda.sh
-bash miniconda.sh -b -p $HOME/miniconda
-export PATH=${HOME}/miniconda/bin:${PATH}
-conda config --set always_yes yes --set changeps1 no
-conda update -y -q conda
-conda config --set anaconda_upload no
-#git clone git://github.com/uv-cdat/uvcdat-testdata
diff --git a/ci-support/conda_upload.sh b/ci-support/conda_upload.sh
index ef61e574..ebde3a90 100644
--- a/ci-support/conda_upload.sh
+++ b/ci-support/conda_upload.sh
@@ -1,83 +1,34 @@
 #!/usr/bin/env bash
 PKG_NAME=cdms2
 USER=cdat
-echo "Trying to upload conda"
-mkdir ${HOME}/conda-bld
-export CONDA_BLD_PATH=${HOME}/conda-bld
-export VERSION="2.12"
+export VERSION="3.0"
+echo "Trying to upload to conda"
+echo ""
+echo "Activating base env"
+source activate base
+echo "Making sure conda-build is installed"
+conda install "conda-build<3.10"
+echo "Updating conda"
+conda update -y -q conda
 if [ `uname` == "Linux" ]; then
     OS=linux-64
     echo "Linux OS"
-    yum install -y wget git gcc
-    # wget --no-check https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh  -O miniconda3.sh 2> /dev/null
-    wget --no-check https://repo.continuum.io/miniconda/Miniconda2-4.3.30-Linux-x86_64.sh  -O miniconda2.sh 2> /dev/null
-    bash miniconda2.sh -b -p ${HOME}/miniconda
-    export SYSPATH=$PATH
-    export PATH=${HOME}/miniconda/bin:${SYSPATH}
-    echo $PATH
-    conda config --set always_yes yes --set changeps1 no
-    conda config --set anaconda_upload false --set ssl_verify false
-    conda install -n root -q anaconda-client "conda-build<3.3"
-    conda install -n root gcc future
-    which python
-    export UVCDAT_ANONYMOUS_LOG=False
-    BRANCH=${TRAVIS_BRANCH}
-#    echo "Creating python 3 env"
-#    conda create -n py3 python=3.6
-#    conda install -n py3 -c conda-forge -c uvcdat setuptools libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient numpy
-#    conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy
-#    echo "Creating certificate"
-#    source activate py3
-#    mkdir ${HOME}/.esg
-#    echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o ${HOME}/.esg/esgf.cert
-#    ls ${HOME}/.esg
-#    cd travis_home
-#    ls
-#    cp tests/dodsrc ${HOME}.dodsrc
-#    source deactivate
 else
     echo "Mac OS"
     OS=osx-64
-    BRANCH=${CIRCLE_BRANCH}
 fi
 
-which python
-if [ `uname` == "Linux" ]; then
-    conda install -n root -q anaconda-client "conda-build<3.3"
-else
-    conda install -n root -q anaconda-client conda-build
-fi
-# pin conda so that conda-build does not update it
-#if [ `uname` == "Darwin" ]; then
-#    echo "conda ==4.3.21" >> ~/miniconda/conda-meta/pinned  # Pin conda as workaround for conda/conda#6030
-#fi
+mkdir ~/conda-bld
 conda config --set anaconda_upload no
+export CONDA_BLD_PATH=${HOME}/conda-bld
 echo "Cloning recipes"
-cd ${HOME}
-git clone git://github.com/UV-CDAT/conda-recipes
+git clone git://github.com/CDAT/conda-recipes
 cd conda-recipes
 # uvcdat creates issues for build -c uvcdat confises package and channel
 rm -rf uvcdat
-python ./prep_for_build.py -b ${BRANCH}
-echo "Building now"
-echo "use nesii/label/dev-esmf for esmf"
-conda build -V
-conda build $PKG_NAME -c nesii/label/dev-esmf  -c uvcdat/label/nightly -c conda-forge -c uvcdat 
-#
-# binstar config set 'false' instead of false (not quote) I have to do it manually
-# this is true for OSX.
-# binstar is changing verify_ssl to ssl_verify, but the later is not always working
-# 
-# binstar config --set verify_ssl false
-# binstar config --set ssl_verify false
-#
-mkdir -p ~/.continuum/anaconda-client/
-echo "ssl_verify: false" >> ~/.continuum/anaconda-client/config.yaml
-echo "verify_ssl: false" >> ~/.continuum/anaconda-client/config.yaml
-if [ `uname` == "Darwin" ]; then
-    # fix conda and anaconda-client conflict
-    conda install conda==4.2.16
-fi
-anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l ${LABEL}  ${CONDA_BLD_PATH}/$OS/$PKG_NAME-$VERSION.`date +%Y`*_0.tar.bz2 --force
-
+export BRANCH=${CIRCLE_BRANCH}
+python ./prep_for_build.py  -b ${BRANCH}
 
+conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge  --python 3.6
+conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge  --python 2.7
+anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL $CONDA_BLD_PATH/$OS/${PKG_NAME}-$VERSION.`date +%Y*`0.tar.bz2 --force
diff --git a/circle.yml b/circle.yml
deleted file mode 100644
index 5d2059a9..00000000
--- a/circle.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-general:
-    #branches:
-    #  ignore:
-      # We only want to build pull requests for testing. If something is merged,
-      # then we are prepping for release an there is no need to build it again.
-      #  - master
-#  artifacts:
-#      - tests_html
-#      - tests_png
-
-checkout:
-  post:
-    - ./ci-support/checkout_merge_commit.sh
-
-#machine:
-machine:
-#    xcode:
-#        version: 7.2
-    pre:
-        - sudo -H pip install --upgrade virtualenv
-        - ls
-        - pwd
-        - bash cdms/ci-support/circleci_mac_machine_pre.sh
-    #services:
-    #    - docker
-
-dependencies:
-    override:
-        - bash ./ci-support/circleci_mac_dep.sh
-          #   - docker pull cdat/conda:conda-forge-cdms2
-  
-test:
-  override:
-      - bash ./ci-support/circleci_mac.sh
-        # - docker run -it -v `pwd`:/git_repo -a STDOUT -a STDERR -P cdat/conda:conda-forge-cdms2 /git_repo/ci-support/circle.sh
diff --git a/tests/dodsrccircleci b/tests/dodsrccircleci
index b9b0608f..b2d0bbe8 100644
--- a/tests/dodsrccircleci
+++ b/tests/dodsrccircleci
@@ -1,6 +1,6 @@
 HTTP.VERBOSE=0
-HTTP.COOKIEJAR=/Users/distiller/.esg/.dods_cookies
-HTTP.SSL.CERTIFICATE=/Users/distiller/.esg/esgf.cert
-HTTP.SSL.KEY=/Users/distiller/.esg/esgf.cert
-HTTP.SSL.CAPATH=/Users/distiller/.esg/
+HTTP.COOKIEJAR=$HOME/.esg/.dods_cookies
+HTTP.SSL.CERTIFICATE=$HOME/.esg/esgf.cert
+HTTP.SSL.KEY=$HOME/.esg/esgf.cert
+HTTP.SSL.CAPATH=$HOME/.esg/
 
diff --git a/tests/test_cdscan.py b/tests/test_cdscan.py
index e53bdcba..97104482 100644
--- a/tests/test_cdscan.py
+++ b/tests/test_cdscan.py
@@ -58,6 +58,7 @@ def testopenFile(self):
             pass
         argv = 'cdscan -x test_dap.xml https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/Reanalysis/NASA-GMAO/GEOS-5/MERRA/mon/atmos/zg.ncml'.split()
         pth = cdat_info.get_sampledata_path()
+
         os.chdir(pth)
         cdscan(argv)
         f=cdms2.open("test_dap.xml")