Cached build of repos when doing TravisCI build

.codecov.yml

@@ -0,0 +1,6 @@
+codecov:
+  token: 7e489333-8f39-4b00-8377-402f46db4a15
+
+ignore:
+  - "/usr/.*"
+  - "/jcsda/work/repo.install/.*"

.travis.yml

@@ -5,128 +5,83 @@ branches:
   only:
     - develop
 
+env:
+  - BUILD_TYPE=Debug
+
 language: cpp
 
 services:
   - docker
 
-before_install:
-  - date
-  - REPO_SOURCE_DIR=${TRAVIS_BUILD_DIR}/jcsda/src_repo
-#  create the same file structure on Travis VM and Docker container
-  - mkdir -p ${TRAVIS_BUILD_DIR}/jcsda/src_repo
-# echo branch info
-  - echo $TRAVIS_PULL_REQUEST
-  - echo $TRAVIS_PULL_REQUEST_BRANCH
-  - echo $TRAVIS_BRANCH
-  - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
-  - echo "TRAVIS_BRANCH=$TRAVIS_BRANCH, PR=$PR, BRANCH=$BRANCH"
-#======================================================================
-# Clone all repos
-#======================================================================
-#  checkout for the same branch name in soca-bundle
-  - git clone https://github.com/jcsda/soca-bundle.git ${REPO_SOURCE_DIR}/soca-bundle
-  - cd ${REPO_SOURCE_DIR}/soca-bundle
-  - git checkout $BRANCH || echo "No branch named $BRANCH in oops repo"
-  - cd $REPO_SOURCE_DIR  
-
-#  clone the testing branch of soca	#  clone the testing branch of soca
-  - git clone -b $BRANCH https://github.com/jcsda/soca.git ${REPO_SOURCE_DIR}/soca-bundle/soca
-
-#  disable git-LFS 
-  - git lfs install --skip-smudge
-
-#  default/develop branch for these repos
-  - git clone https://github.com/jcsda/GSW-Fortran.git ${REPO_SOURCE_DIR}/soca-bundle/gsw
-  - git clone https://github.com/jcsda/fckit.git ${REPO_SOURCE_DIR}/soca-bundle/fckit
-  - git clone https://github.com/jcsda/soca-config.git ${REPO_SOURCE_DIR}/soca-bundle/soca-config
-
-#  specific branch for these repo
-  - git clone -b dev/master-ecbuild https://github.com/JCSDA/FMS.git ${REPO_SOURCE_DIR}/soca-bundle/fms
-  - git clone -b feature/ecbuild  https://github.com/JCSDA/CVMix-src.git ${REPO_SOURCE_DIR}/soca-bundle/cvmix
-  - git clone -b feature/ecbuild  https://github.com/JCSDA/geoKdTree.git ${REPO_SOURCE_DIR}/soca-bundle/geokdtree
-  - git clone -b feature/ecbuild  https://github.com/JCSDA/MOM6_DA_hooks.git ${REPO_SOURCE_DIR}/soca-bundle/mom6_da_hooks
-  - git clone -b dev/master-ecbuild https://github.com/JCSDA/MOM6.git ${REPO_SOURCE_DIR}/soca-bundle/mom6
-
-#  checkout for the same branch name in oops
-  - git clone https://github.com/jcsda/oops.git ${REPO_SOURCE_DIR}/soca-bundle/oops
-  - cd ${REPO_SOURCE_DIR}/soca-bundle/oops
-  - git checkout $BRANCH || echo "No branch named $BRANCH in oops repo"
-  - cd $REPO_SOURCE_DIR  
-  - date  
-
-#  checkout for the same branch name in crtm
-  - date
-  - git clone https://github.com/jcsda/crtm.git ${REPO_SOURCE_DIR}/soca-bundle/crtm
-  - cd ${REPO_SOURCE_DIR}/soca-bundle/crtm
-  - git checkout $BRANCH || echo "No branch named $BRANCH in crtm repo"
-  - cd $REPO_SOURCE_DIR
-  - date  
+cache:
+  ccache: true
+  directories:
+  - $HOME/repo.cache
 
-#  checkout for the same branch name in ioda
-  - git clone https://github.com/jcsda/ioda.git ${REPO_SOURCE_DIR}/soca-bundle/ioda
-  - cd ${REPO_SOURCE_DIR}/soca-bundle/ioda
-  - git checkout $BRANCH || echo "No branch named $BRANCH in ioda repo"
-  - cd $REPO_SOURCE_DIR  
-  - date  
-
-#  checkout for the same branch name in ioda-converters
-  - git clone https://github.com/jcsda/ioda-converters.git ${REPO_SOURCE_DIR}/soca-bundle/ioda-conventers
-  - cd ${REPO_SOURCE_DIR}/soca-bundle/ioda-conventers
-  - git checkout $BRANCH || echo "No branch named $BRANCH in ioda repo"
-  - cd $REPO_SOURCE_DIR  
-  - date  
-
-#  checkout for the same branch name in ufo
-  - git clone https://github.com/jcsda/ufo.git ${REPO_SOURCE_DIR}/soca-bundle/ufo
-  - cd ${REPO_SOURCE_DIR}/soca-bundle/ufo
-  - git checkout $BRANCH || echo "No branch named $BRANCH in ioda repo"
-  - cd $REPO_SOURCE_DIR  
-  - date  
-
-#  checkout for the same branch name in saber
-  - git clone https://github.com/jcsda/saber.git ${REPO_SOURCE_DIR}/soca-bundle/saber
-  - cd ${REPO_SOURCE_DIR}/soca-bundle/saber
-  - git checkout $BRANCH || echo "No branch named $BRANCH in saber repo"
-  - cd $REPO_SOURCE_DIR  
-  - date  
-
-  - ls ${REPO_SOURCE_DIR}/soca-bundle/
-
-# not switch to develop branch by removing 
-# develop UPDATE in CMakeLists.txt in soca-bundle
-
-  - cd soca-bundle
-  - sed -i 's/UPDATE BRANCH develop //' CMakeLists.txt
-  - cd ${REPO_SOURCE_DIR}
-  - ls ${REPO_SOURCE_DIR}
-  - ls ${REPO_SOURCE_DIR}/soca-bundle
-  - find . -name \* -exec chmod 777 {} \;
 
 #======================================================================
-# get docker image
-# FROM  jcsda/docker:latest
 #======================================================================
-  - cd ${REPO_SOURCE_DIR}/soca-bundle/soca
-  - date
-  - docker build -t jcsda/dockerl2 --build-arg=Dockerfile .  #Dockerfile handles ssh for mpi
-  - date
-  - docker images
-  # request codecov to detect CI environment to pass through to docker
-  - ci_env=`bash <(curl -s https://codecov.io/env)`
-  - docker run -d -t $ci_env --name jcsda_container -v ${REPO_SOURCE_DIR}/soca-bundle:/jcsda/src_repo jcsda/dockerl2
-  - docker ps -a
-  - date
+before_install:
+  - |
+    # setup the configurables that the subsequent scripts use
+    export BUNDLE_URL="https://github.com/JCSDA/soca-bundle.git"
+    export MAIN_REPO=soca
+    export MAIN_BUILD_TYPE=${BUILD_TYPE}    
+    export LIB_REPOS="fms cvmix geokdtree mom6_da_hooks gsw mom6
+                      fckit saber oops ioda ufo ioda-converters soca-config"
+    export LIB_BUILD_TYPE=RelWIthDebInfo
+    export LIB_BUILD_OPT_OOPS="-DENABLE_OOPS_TOYMODELS=OFF"
+    export MATCH_REPOS="saber oops ioda ioda-converters ufo soca soca-config"
+    export LFS_REPOS="soca"
+    export BRANCH=$( [[ "$TRAVIS_PULL_REQUEST" == "false" ]] && \
+                     echo $TRAVIS_BRANCH || echo $TRAVIS_PULL_REQUEST_BRANCH )
+
+    export_vars="-e REPO_CACHE=/repo.cache -e LIB_REPOS -e LIB_BUILD_TYPE 
+                 -e MAIN_REPO -e MAIN_BUILD_TYPE -e CCACHE_DIR=/ccache
+                 -e LIB_BUILD_OPT_OOPS"
+
+  - |
+    # clone and prep the repos
+    WORK_DIR=$HOME/work
+    mkdir -p $WORK_DIR; cd $WORK_DIR
+    REPO_CACHE=$HOME/repo.cache ${TRAVIS_BUILD_DIR}/.travis/prep.sh
+    mv ${TRAVIS_BUILD_DIR} ${WORK_DIR}/repo.src/
+
+  - |
+    # alter file permissions
+    chmod 777 -R ~/.ccache
+    chmod 777 -R $HOME/repo.cache
+    cd ${WORK_DIR} && find . -name \* -exec chmod 777 {} \;
+
+  - |
+    # prepare docker image from jcsda/docker:latest
+    cd ${WORK_DIR}/repo.src/${MAIN_REPO}/.travis
+    docker build -t jcsda/dockerl2 --build-arg=Dockerfile .
+    docker images
+    ci_env=`bash <(curl -s https://codecov.io/env)`
+    docker run -d -t $ci_env $export_vars \
+      -v ${WORK_DIR}:/jcsda/work \
+      -v $HOME/.ccache:/ccache \
+      -v $HOME/repo.cache:/repo.cache \
+      --name jcsda_container jcsda/dockerl2
+    docker ps -a
+
+
 #======================================================================
 # Here are the run steps
 #======================================================================
 script:
-  - docker exec jcsda_container ls
-  - docker exec jcsda_container ls /jcsda/src_repo
-  - docker exec jcsda_container bash -c 'cp -r /jcsda/.openmpi/ ~/'
-  - docker exec jcsda_container bash -c 'cd /build_container && ecbuild /jcsda/src_repo/ -DSOCA_TESTS_FORC_DEFAULT_TOL=ON' 
-  - docker exec jcsda_container bash -c 'cd /build_container/soca && make -j4'
-  - date
-  - docker exec jcsda_container bash -c 'cd /build_container/soca && ctest -V'
-  - date
-
+  - |
+    # Build code
+    docker exec jcsda_container bash \
+     -c 'cd /jcsda/work && /jcsda/work/repo.src/soca/.travis/build.sh'
+
+  - |
+    # run tests
+    docker exec jcsda_container bash \
+     -c 'cd /jcsda/work/repo.build/soca && ctest --output-on-failure'
+
+  - |
+    # upload code coverage
+    docker exec jcsda_container bash \
+     -c 'cd /jcsda/work/repo.src/soca && lcov -c -d . -d ../../repo.build/soca/ -o coverage.info  --no-external -q && bash <(curl -s https://codecov.io/bash)'

.travis/Dockerfile

@@ -0,0 +1,37 @@
+FROM  jcsda/docker:latest
+
+RUN cd /usr/local/src \
+    && curl -L -O https://github.com/ccache/ccache/releases/download/v3.7.4/ccache-3.7.4.tar.gz \
+    && tar -xaf ccache-3.7.4.tar.gz \
+    && cd ccache-3.7.4 \
+    && ./configure \
+    && make \
+    && make install
+
+RUN mkdir -p /var/run/sshd \
+    && ssh-keygen -A \
+    && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config
+
+RUN groupadd jcsda -g 9999 \
+    && adduser jcsdauser \
+    && mkdir -p /jcsda \
+    && chown -R jcsdauser:jcsda /jcsda \
+    && chmod 6755 /jcsda 
+
+USER jcsdauser
+
+COPY default-mca-params.conf /jcsda/mca-params.conf
+
+RUN mkdir -p ~/.openmpi \
+    && cp /jcsda/mca-params.conf ~/.openmpi/
+
+RUN mkdir /jcsda/.ssh \
+    && echo "StrictHostKeyChecking no" > /jcsda/.ssh/config \
+    && ssh-keygen -f /jcsda/.ssh/id_rsa -t rsa -N '' \
+    && chmod 600 /jcsda/.ssh/config \
+    && chmod 700 /jcsda/.ssh \
+    && cp /jcsda/.ssh/id_rsa.pub /jcsda/.ssh/authorized_keys
+
+CMD ["/bin/bash"]

.travis/build.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -e
+
+cwd=$(pwd)
+
+# zero out the ccache stats
+echo -e "\nzeroing out 'ccache' statistics"
+ccache -z
+
+# for each dependency repo, make install, if needed
+for repo in $LIB_REPOS; do
+    bundle_dir=$cwd/repo.bundle/$repo
+    src_dir=$cwd/repo.src/$repo
+    build_dir=$cwd/repo.build/$repo
+    install_dir=${REPO_CACHE}/$repo
+
+    # set the path for the install dir for subsequent repos to find
+    repo_upper=${repo/-/_}; repo_upper=${repo_upper^^}
+    typeset "${repo_upper}_PATH=$install_dir"
+    export ${repo_upper}_PATH
+
+    # do we skip building this repo?
+    [[ -e $bundle_dir/skip_rebuild ]] && continue
+
+    # if we are to build
+    echo -e "\n"
+    echo "************************************************************"
+    echo "  $repo"
+    echo "************************************************************"
+    rm -rf $build_dir
+    rm -rf $install_dir
+    mkdir -p $build_dir
+    cd $build_dir
+
+    # run ecbuild
+    build_opt_var=LIB_BUILD_OPT_${repo_upper}
+    build_opt=${!build_opt_var}
+    time ecbuild $src_dir -DCMAKE_INSTALL_PREFIX=${install_dir} -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DENABLE_TESTS=OFF -DBUILD_TESTING=OFF $build_opt
+
+    # build and install
+    time make -j4
+    time make install
+
+    # save compilation info
+    cp $bundle_dir/build.version $install_dir/
+done
+
+
+# build the main test repo
+echo -e "\n"
+echo "************************************************************"
+echo "  $MAIN_REPO"
+echo "************************************************************"
+cd $cwd
+src_dir=$cwd/repo.src/${MAIN_REPO}
+build_dir=$cwd/repo.build/${MAIN_REPO}
+mkdir -p  $build_dir
+cd $build_dir
+time ecbuild $src_dir -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=${MAIN_BUILD_TYPE} -DENABLE_GPROF=ON
+time make -j4
+
+
+# how useful was ccache?
+echo -e "\nccache statistics:"
+ccache -s