diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 8a582e3b..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,147 +0,0 @@ -version: 2 - -checkout: - post: - - ./ci-support/checkout_merge_commit.sh - -aliases: - - - &setup_miniconda - name: setup_miniconda - command: | - mkdir -p workspace - git clone -b validateNightly git@github.com:CDAT/cdat workspace/cdat - ls workspace/cdat - # following will install miniconda3 under $WORKDIR/miniconda/bin - python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3' - - - &create_conda_env - name: create_conda_env - command: | - export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH - conda config --set always_yes yes --set changeps1 no - conda update -y -q conda - conda config --set anaconda_upload no - conda create -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient "python>3" - conda create -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 "python<3" - if [ $(uname) == "Linux" ]; then - conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64 - conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64 - else - conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc - conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc - fi - - - &setup_cdms - name: setup_cdms - command: | - export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH - export UVCDAT_ANONYMOUS_LOG=False - source activate py3 - mkdir $HOME/.esg - echo "Get ESGF certificates" - echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o $HOME/.esg/esgf.cert - cp tests/dodsrccircleci $HOME/.dodsrc - echo "Create .dods_cookies" - curl -L -v -c $HOME/.esg/.dods_cookies --cert $HOME/.esg/esgf.cert --key $HOME/.esg/esgf.cert "https://aims3.llnl.gov/thredds/dodsC/cmip5_css02_data/cmip5/output1/CMCC/CMCC-CM/decadal2005/mon/atmos/Amon/r1i1p1/cct/1/cct_Amon_CMCC-CM_decadal2005_r1i1p1_202601-203512.nc.dds" - if [ $(uname) == "Linux" ];then - export LDSHARED="$CC -shared -pthread" - LDSHARED="$CC -shared -pthread" python setup.py install - else - python setup.py install - fi - source activate py2 - rm -rf build - if [ $(uname) == "Linux" ];then - export LDSHARED="$CC -shared -pthread" - LDSHARED="$CC -shared -pthread" python setup.py install - else - python setup.py install - fi - - - &run_cdms_tests - name: run_cdms_tests - command: | - export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH - export UVCDAT_ANONYMOUS_LOG=False - set -e - source activate py2 - python run_tests.py -v2 - PY2_RESULT=$? - echo "*** py2 test result: "${PY2_RESULT} - source activate py3 - python run_tests.py -v2 - PY3_RESULT=$? - echo "*** py3 test result: "${PY3_RESULT} - echo $PY2_RESULT > $HOME/project/$WORKDIR/py2_result.txt - echo $PY3_RESULT > $HOME/project/$WORKDIR/py3_result.txt - - - &upload_cdms - name: upload_cdms - command: | - export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH - export LABEL="nightly"; - # Retrieve results - PY2_RESULT=$(cat $HOME/project/$WORKDIR/py2_result.txt) - PY3_RESULT=$(cat $HOME/project/$WORKDIR/py3_result.txt) - echo "*** CIRCLE BRANCH: "${CIRCLE_BRANCH} - echo "*** PY2_RESULT: "${PY2_RESULT} - echo "*** PY3_RESULT: "${PY3_RESULT} - if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then bash ./ci-support/conda_upload.sh ; fi - if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then conda install -n root conda-build anaconda-client ; fi - #if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH != "master" ]; then - if [ $CIRCLE_BRANCH != "master" ]; then - export LABEL="unstable"; - echo "NOTE: upload LABEL="${LABEL} - conda install -n root conda-build anaconda-client; - bash ./ci-support/conda_upload.sh; - fi - -jobs: - macos_cdms: - macos: - xcode: "9.2.0" - environment: - WORKDIR: "workspace/test_macos_cdms" - steps: - - checkout - - run: *setup_miniconda - - run: *create_conda_env - - run: *setup_cdms - - run: *run_cdms_tests - - run: *upload_cdms - - store_artifacts: - path: tests_html - destination: tests_html - - store_artifacts: - path: tests_png - destination: tests_png - - linux_cdms: - machine: - image: circleci/classic:latest - environment: - WORKDIR: "workspace/test_linux_cdms" - steps: - - checkout - - run: *setup_miniconda - - run: *create_conda_env - - run: *setup_cdms - - run: *run_cdms_tests - - run: *upload_cdms - - store_artifacts: - path: tests_html - destination: tests_html - - store_artifacts: - path: tests_png - destination: tests_png - - -workflows: - version: 2 - cdms_test: - jobs: - - macos_cdms - - linux_cdms - - diff --git a/.travis.yml b/.travis.yml index 0bb331b0..26d87cc8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ before_install: script: #- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 - conda create -n py3 python=3.6 -- conda install -n py3 -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient +- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient - source activate py3 - if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy ; fi - export UVCDAT_ANONYMOUS_LOG=False @@ -37,7 +37,7 @@ script: - conda create -n py2 python=2.7 - source activate py2 - which python -- conda install -c nesii/channel/dev-esmf -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 +- conda install -c nesii/channel/dev-esmf -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 - export UVCDAT_ANONYMOUS_LOG=False - python setup.py install - python run_tests.py -v2 -s diff --git a/ci-support/circleci_mac.sh b/ci-support/circleci_mac.sh new file mode 100644 index 00000000..66e14816 --- /dev/null +++ b/ci-support/circleci_mac.sh @@ -0,0 +1,15 @@ +export UVCDAT_ANONYMOUS_LOG=False +export PATH=${HOME}/miniconda/bin:${PATH} +echo "CIRCLE CI BRANCH:"$CIRCLE_BRANCH +echo "CI_PULL_REQUESTS"$CI_PULL_REQUESTS +echo "CI_PULL_REQUEST"$CI_PULL_REQUEST +source activate py2 +python run_tests.py -v2 -s +RESULT=$? +source activate py3 +python run_tests.py -v2 -s +RESULT=$(( $RESULT + $? )) +echo "RESULT:"${RESULT} +if [ $RESULT -eq 0 -a $CIRCLE_BRANCH != "master" ]; then LABEL='unstable' bash ./ci-support/conda_upload.sh ; fi +if [ $RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then LABEL='nightly' bash ./ci-support/conda_upload.sh ; fi +exit $RESULT diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh new file mode 100644 index 00000000..385aef61 --- /dev/null +++ b/ci-support/circleci_mac_dep.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +cmd="ls" +echo $cmd +$cmd + +cmd="pwd" +echo $cmd +$cmd + +cmd="export PATH=${HOME}/miniconda/bin:${PATH}" +echo $cmd +$cmd + +# Create Python 3 environment +cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=5" +echo $cmd +$cmd + +# update openssl for myproxyclient +cmd="conda install -n py3 pyopenssl" +echo $cmd +$cmd + + +cmd="conda install -n py3 -c nesii/label/dev-esmf -c conda-forge esmf esmpy netcdf-fortran=4.4.4=5" +echo $cmd +$cmd + +# Create Python 2 environment +cmd="conda create -n py2 python=2.7" +echo $cmd +$cmd + +# Activate python 2 environment +cmd="source activate py2" +echo $cmd +$cmd + +cmd="conda install -n py2 -c nesii/label/dev-esmf -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=5" +echo $cmd +$cmd + +# update openssl for myproxyclient +cmd="conda install -n py2 pyopenssl" +echo $cmd +$cmd +: + +# add relative path to ncdump +#cmd="install_name_tool -change /usr/lib/libcurl.4.dylib @rpath/libcurl.4.dylib ${HOME}/miniconda/envs/py2/bin/ncdump" +#echo $cmd +#$cmd + +#cmd="install_name_tool -change /usr/lib/libcurl.4.dylib @rpath/libcurl.4.dylib ${HOME}/miniconda/envs/py3/bin/ncdump" +#echo $cmd +#$cmd + +cmd="export UVCDAT_ANONYMOUS_LOG=False" +echo $cmd +$cmd + +# Retrieve certificates from ESGF +cmd="mkdir /Users/distiller/.esg" +echo $cmd +$cmd + +cmd="echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o /Users/distiller/.esg/esgf.cert " +eval $cmd + +echo "Create .dods_cookies" +cmd="curl -L -v -c /Users/distiller/.esg/.dods_cookies --cert /Users/distiller/.esg/esgf.cert --key /Users/distiller/.esg/esgf.cert https://aims3.llnl.gov/thredds/dodsC/cmip5_css02_data/cmip5/output1/CMCC/CMCC-CM/decadal2005/mon/atmos/Amon/r1i1p1/cct/1/cct_Amon_CMCC-CM_decadal2005_r1i1p1_202601-203512.nc.dds" +echo $cmd +$cmd + +#cmd="openssl pkcs12 -export -inkey /Users/distiller/.esg/esgf.cert -in /Users/distiller/.esg/esgf.cert -name esgf -out /Users/distiller/.esg/esgf.p12 -passout pass:esgf" +#$cmd + +#cmd="sudo security import /Users/distiller/.esg/esgf.p12 -A -P esgf -k /Library/Keychains/System.keychain" +#echo $cmd +#$cmd + +#cmd="sudo security add-trusted-cert -d -r trustRoot -k '/Library/Keychains/System.keychain' /Users/distiller/.esg/esgf.cert" +#echo $cmd +#$cmd + +#cmd="cp tests/dodsrccircleci /Users/distiller/.dodsrc" +#echo $cmd +#$cmd + +# compile cdms on py2 and py3 environemt. + +cmd="python setup.py install" +echo $cmd +$cmd + +cmd="source activate py3" +echo $cmd +$cmd + +cmd="python setup.py install" +echo $cmd +$cmd diff --git a/ci-support/circleci_mac_machine_pre.sh b/ci-support/circleci_mac_machine_pre.sh new file mode 100644 index 00000000..386629ec --- /dev/null +++ b/ci-support/circleci_mac_machine_pre.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +curl https://repo.continuum.io/miniconda/Miniconda3-4.3.30.1-MacOSX-x86_64.sh -o miniconda.sh +bash miniconda.sh -b -p $HOME/miniconda +export PATH=${HOME}/miniconda/bin:${PATH} +conda config --set always_yes yes --set changeps1 no +conda update -y -q conda +conda config --set anaconda_upload no +#git clone git://github.com/uv-cdat/uvcdat-testdata diff --git a/ci-support/conda_upload.sh b/ci-support/conda_upload.sh index ebde3a90..ef61e574 100644 --- a/ci-support/conda_upload.sh +++ b/ci-support/conda_upload.sh @@ -1,34 +1,83 @@ #!/usr/bin/env bash PKG_NAME=cdms2 USER=cdat -export VERSION="3.0" -echo "Trying to upload to conda" -echo "" -echo "Activating base env" -source activate base -echo "Making sure conda-build is installed" -conda install "conda-build<3.10" -echo "Updating conda" -conda update -y -q conda +echo "Trying to upload conda" +mkdir ${HOME}/conda-bld +export CONDA_BLD_PATH=${HOME}/conda-bld +export VERSION="2.12" if [ `uname` == "Linux" ]; then OS=linux-64 echo "Linux OS" + yum install -y wget git gcc + # wget --no-check https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda3.sh 2> /dev/null + wget --no-check https://repo.continuum.io/miniconda/Miniconda2-4.3.30-Linux-x86_64.sh -O miniconda2.sh 2> /dev/null + bash miniconda2.sh -b -p ${HOME}/miniconda + export SYSPATH=$PATH + export PATH=${HOME}/miniconda/bin:${SYSPATH} + echo $PATH + conda config --set always_yes yes --set changeps1 no + conda config --set anaconda_upload false --set ssl_verify false + conda install -n root -q anaconda-client "conda-build<3.3" + conda install -n root gcc future + which python + export UVCDAT_ANONYMOUS_LOG=False + BRANCH=${TRAVIS_BRANCH} +# echo "Creating python 3 env" +# conda create -n py3 python=3.6 +# conda install -n py3 -c conda-forge -c uvcdat setuptools libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient numpy +# conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy +# echo "Creating certificate" +# source activate py3 +# mkdir ${HOME}/.esg +# echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o ${HOME}/.esg/esgf.cert +# ls ${HOME}/.esg +# cd travis_home +# ls +# cp tests/dodsrc ${HOME}.dodsrc +# source deactivate else echo "Mac OS" OS=osx-64 + BRANCH=${CIRCLE_BRANCH} fi -mkdir ~/conda-bld +which python +if [ `uname` == "Linux" ]; then + conda install -n root -q anaconda-client "conda-build<3.3" +else + conda install -n root -q anaconda-client conda-build +fi +# pin conda so that conda-build does not update it +#if [ `uname` == "Darwin" ]; then +# echo "conda ==4.3.21" >> ~/miniconda/conda-meta/pinned # Pin conda as workaround for conda/conda#6030 +#fi conda config --set anaconda_upload no -export CONDA_BLD_PATH=${HOME}/conda-bld echo "Cloning recipes" -git clone git://github.com/CDAT/conda-recipes +cd ${HOME} +git clone git://github.com/UV-CDAT/conda-recipes cd conda-recipes # uvcdat creates issues for build -c uvcdat confises package and channel rm -rf uvcdat -export BRANCH=${CIRCLE_BRANCH} -python ./prep_for_build.py -b ${BRANCH} +python ./prep_for_build.py -b ${BRANCH} +echo "Building now" +echo "use nesii/label/dev-esmf for esmf" +conda build -V +conda build $PKG_NAME -c nesii/label/dev-esmf -c uvcdat/label/nightly -c conda-forge -c uvcdat +# +# binstar config set 'false' instead of false (not quote) I have to do it manually +# this is true for OSX. +# binstar is changing verify_ssl to ssl_verify, but the later is not always working +# +# binstar config --set verify_ssl false +# binstar config --set ssl_verify false +# +mkdir -p ~/.continuum/anaconda-client/ +echo "ssl_verify: false" >> ~/.continuum/anaconda-client/config.yaml +echo "verify_ssl: false" >> ~/.continuum/anaconda-client/config.yaml +if [ `uname` == "Darwin" ]; then + # fix conda and anaconda-client conflict + conda install conda==4.2.16 +fi +anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l ${LABEL} ${CONDA_BLD_PATH}/$OS/$PKG_NAME-$VERSION.`date +%Y`*_0.tar.bz2 --force + -conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge --python 3.6 -conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge --python 2.7 -anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL $CONDA_BLD_PATH/$OS/${PKG_NAME}-$VERSION.`date +%Y*`0.tar.bz2 --force diff --git a/circle.yml b/circle.yml new file mode 100644 index 00000000..5d2059a9 --- /dev/null +++ b/circle.yml @@ -0,0 +1,35 @@ +general: + #branches: + # ignore: + # We only want to build pull requests for testing. If something is merged, + # then we are prepping for release an there is no need to build it again. + # - master +# artifacts: +# - tests_html +# - tests_png + +checkout: + post: + - ./ci-support/checkout_merge_commit.sh + +#machine: +machine: +# xcode: +# version: 7.2 + pre: + - sudo -H pip install --upgrade virtualenv + - ls + - pwd + - bash cdms/ci-support/circleci_mac_machine_pre.sh + #services: + # - docker + +dependencies: + override: + - bash ./ci-support/circleci_mac_dep.sh + # - docker pull cdat/conda:conda-forge-cdms2 + +test: + override: + - bash ./ci-support/circleci_mac.sh + # - docker run -it -v `pwd`:/git_repo -a STDOUT -a STDERR -P cdat/conda:conda-forge-cdms2 /git_repo/ci-support/circle.sh diff --git a/tests/dodsrccircleci b/tests/dodsrccircleci index b2d0bbe8..b9b0608f 100644 --- a/tests/dodsrccircleci +++ b/tests/dodsrccircleci @@ -1,6 +1,6 @@ HTTP.VERBOSE=0 -HTTP.COOKIEJAR=$HOME/.esg/.dods_cookies -HTTP.SSL.CERTIFICATE=$HOME/.esg/esgf.cert -HTTP.SSL.KEY=$HOME/.esg/esgf.cert -HTTP.SSL.CAPATH=$HOME/.esg/ +HTTP.COOKIEJAR=/Users/distiller/.esg/.dods_cookies +HTTP.SSL.CERTIFICATE=/Users/distiller/.esg/esgf.cert +HTTP.SSL.KEY=/Users/distiller/.esg/esgf.cert +HTTP.SSL.CAPATH=/Users/distiller/.esg/ diff --git a/tests/test_cdscan.py b/tests/test_cdscan.py index 97104482..e53bdcba 100644 --- a/tests/test_cdscan.py +++ b/tests/test_cdscan.py @@ -58,7 +58,6 @@ def testopenFile(self): pass argv = 'cdscan -x test_dap.xml https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/Reanalysis/NASA-GMAO/GEOS-5/MERRA/mon/atmos/zg.ncml'.split() pth = cdat_info.get_sampledata_path() - os.chdir(pth) cdscan(argv) f=cdms2.open("test_dap.xml")