Merge branch 'master' of https://github.com/dmlc/mxnet into iter_exc_…

…handling_fix

.clang-tidy

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# The checks defined here will be run and will display by default as warnings.
+Checks: >
+    -*, cppcoreguidelines-c-copy-assignment-signature,
+    cppcoreguidelines-interfaces-global-init, cppcoreguidelines-no-malloc,
+    cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-pro-type-const-cast,
+    cppcoreguidelines-pro-type-cstyle-cast, cppcoreguidelines-pro-type-member-init,
+    cppcoreguidelines-pro-type-static-cast-downcast, cppcoreguidelines-pro-type-union-access,
+    cppcoreguidelines-pro-type-vararg, cppcoreguidelines-slicing,
+    cppcoreguidelines-special-member-functions, clang-analyzer-security.FloatLoopCounter,
+    clang-analyzer-security.insecureAPI.*, clang-analyzer-core.CallAndMessage,
+    clang-analyzer-core.DivideZero, clang-analyzer-core.DynamicTypePropagation,
+    clang-analyzer-core.NonNullParamChecker, clang-analyzer-core.NullDereference,
+    clang-analyzer-core.StackAddressEscape, clang-analyzer-core.UndefinedBinaryOperatorResult,
+    clang-analyzer-core.VLASize, clang-analyzer-core.builtin.BuiltinFunctions,
+    clang-analyzer-core.builtin.NoReturnFunctions, clang-analyzer-core.uninitialized.ArraySubscript,
+    clang-analyzer-core.uninitialized.Assign, clang-analyzer-core.uninitialized.Branch,
+    clang-analyzer-core.uninitialized.CapturedBlockVariable,
+    clang-analyzer-core.uninitialized.UndefReturn, clang-analyzer-cplusplus.NewDelete,
+    clang-analyzer-cplusplus.NewDeleteLeaks, clang-analyzer-cplusplus.SelfAssignment,
+    clang-analyzer-deadcode.DeadStores, modernize-avoid-bind, modernize-deprecated-headers,
+    modernize-loop-convert, modernize-make-shared, modernize-pass-by-value,
+    modernize-raw-string-literal, modernize-redundant-void-arg, modernize-replace-auto-ptr,
+    modernize-replace-random-shuffle, modernize-return-braced-init-list, modernize-shrink-to-fit,
+    modernize-unary-static-assert, modernize-use-bool-literals, modernize-use-default-member-init,
+    modernize-use-emplace, modernize-use-equals-default, modernize-use-equals-delete,
+    modernize-use-noexcept, modernize-use-nullptr, modernize-use-override,
+    modernize-use-transparent-functors, modernize-use-using, performance-*
+
+# cppcoreguidelines checks not enabled:
+# cppcoreguidelines-pro-bounds-pointer-arithmetic
+# cppcoreguidelines-pro-bounds-array-to-pointer-decay
+# cppcoreguidelines-pro-type-reinterpret-cast
+
+# modernize checks not enabled:
+# modernize-use-auto
+# modernize-make-unique (C++14 and newer only)
+
+# In order to trigger an error, you must have a rule defined both in checks and in this section.
+WarningsAsErrors: >
+    cppcoreguidelines-no-malloc, performance-unnecessary-copy-initialization
+
+# Todo: define a better regex match that includes most project headers, but excludes third party
+# code.
+HeaderFilterRegex: '^src/.*'

.travis.yml

@@ -1,108 +1,31 @@
-sudo: false
+sudo: true
 
 language: cpp
 
+cache: ccache
+
 os:
-  # - linux
   - osx
 
-osx_image: xcode8
-
-env:
-  # code analysis
-  # - TASK=lint
-  # build mxnet.so with CUDA
-  # - TASK=build
-  # run tests/cpp
-  - TASK=cpp_test
-  # run tests/python
-  - TASK=python_test
-  - TASK=r_test
-  # - TASK=julia JULIA_VER=0.4
-  # - TASK=scala_test
-
-  # TODO, R test, distributed test, clang, more g++ versions
+osx_image: xcode9.4
 
 matrix:
   include:
-    - # os: linux
-      # dist: trusty
-      # env: TASK=perl_test
     - os: osx
-      ## sudo is required because
-      ## prexexisting packages conflict
-      ## with new ones.
-      ## would be nice to have macports
-      ## on travis osx, it has all needed perl packages
-      sudo: required
-      env: TASK=perl_test
-#       env: TASK=julia JULIA_VER=0.4
-#     - os: linux
-#       env: TASK=build
-#     - os: linux
-#       env: TASK=cpp_test
-#     - os: linux
-#       env: TASK=python_test
-#     - os: linux
-#       env: TASK=r_test
-#     - os: linux
-#       env: TASK=scala_test
-
-# dependent apt packages
-addons:
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - doxygen
-      - wget
-      - git
-      - libcurl4-openssl-dev
-      - unzip
-      - libatlas-dev
-      - libopencv-dev
-      - gcc-4.8
-      - g++-4.8
-      - python-numpy
-      - python-nose
-      - python3-numpy
-      - python3-dev
-      - python3-nose
-      - python-h5py
-      - python3-h5py
-      - graphviz
-      - libmouse-perl
-      - pdl
-      - cpanminus
-      - swig
-      - libgraphviz-perl
 
 before_install:
-  - export NVCC_PREFIX=${HOME}
-  - source dmlc-core/scripts/travis/travis_setup_env.sh
   - export PYTHONPATH=${PYTHONPATH}:${PWD}/python
-  - export MAVEN_SKIP_RC=true
-  - export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=256m -XX:-UseGCOverheadLimit -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
 
 install:
-  - source tests/travis/setup.sh
-
+  - brew install ccache
+  - export PATH="/usr/local/opt/ccache/libexec:$PATH"
+  - source ci/travis/install.sh
+
+# We build with 2 concurrent jobs to match the number of cores present on MacOS virutal machines.
+# nproc does not report the correct number of cores reliably in Travis, so using nproc is not
+# recommended.
+# https://docs.travis-ci.com/user/reference/overview/
 script:
-  - tests/travis/run_test.sh
-
-cache:
-  directories:
-    - ${HOME}/.cache/usr
-
-before_cache:
-  - dmlc-core/scripts/travis/travis_before_cache.sh
-
-after_failure:
-  - tests/travis/travis_after_failure.sh
-
-notifications:
-# Emails are sent to the committer's git-configured email address by default,
-  email:
-    on_success: change
-    on_failure: always
-  #slack: dmlc:NmroCzntCiWOuxUZpii40USd
+  - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+  - mv make/osx.mk config.mk
+  - make -j 2

CMakeLists.txt

@@ -131,7 +131,8 @@ else(MSVC)
   else()
     set(SUPPORT_F16C FALSE)
   endif()
-  set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -fPIC -Wno-sign-compare")
+  set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+  set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -Wno-sign-compare")
   if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$")
     set(CMAKE_C_FLAGS "-Wno-braced-scalar-init")
   endif()

CODEOWNERS

@@ -12,10 +12,11 @@
 *			@apache/mxnet-committers
 
 # Language bindings
-/R-package/       @thirdwing
-/scala-package/   @yzhliu @nswamy
-/perl-package/    @sergeykolychev
-/python/          @szha
+/R-package/                @thirdwing
+/scala-package/            @yzhliu @nswamy
+/perl-package/             @sergeykolychev
+/python/                   @szha
+/contrib/clojure-package/  @gigasquid
 
 # C++ base
 /src/kvstore/     @rahul003 @anirudh2290

CONTRIBUTORS.md

@@ -178,3 +178,5 @@ List of Contributors
 * [Aaron Markham](https://github.com/aaronmarkham)
 * [Sam Skalicky](https://github.com/samskalicky)
 * [Per Goncalves da Silva](https://github.com/perdasilva)
+* [Zhijingcheng Yu](https://github.com/jasonyu1996)
+* [Cheng-Che Lee](https://github.com/stu1130)

Jenkinsfile

@@ -173,12 +173,22 @@ core_logic: {
         }
       }
     },
-    'CPU: Clang 5': {
+    'CPU: Clang 6': {
       node(NODE_LINUX_CPU) {
-        ws('workspace/build-cpu-clang50') {
+        ws('workspace/build-cpu-clang60') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
-            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50', false)
+            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang60', false)
+          }
+        }
+      }
+    },
+    'CPU: Clang Tidy': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/build-cpu-clang60_tidy') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.init_git()
+            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang_tidy', false)
           }
         }
       }
@@ -194,13 +204,13 @@ core_logic: {
         }
       }
     },
-    'CPU: Clang 5 MKLDNN': {
+    'CPU: Clang 6 MKLDNN': {
       node(NODE_LINUX_CPU) {
-        ws('workspace/build-cpu-mkldnn-clang50') {
+        ws('workspace/build-cpu-mkldnn-clang60') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
-            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50_mkldnn', false)
-            utils.pack_lib('mkldnn_cpu_clang5', mx_mkldnn_lib)
+            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang60_mkldnn', false)
+            utils.pack_lib('mkldnn_cpu_clang6', mx_mkldnn_lib)
           }
         }
       }

KEYS

@@ -539,3 +539,63 @@ C0s/d2Nj+1o4aisymgm9e87IAkDc3eXCo2WMHPkR89nOs/vYpyMrlobKcyCvsDC/
 hSr9ITOiZCc=
 =VCxR
 -----END PGP PUBLIC KEY BLOCK-----
+
+pub   rsa4096 2018-08-23 [SC]
+      CB9164C76A803D861DE16E5B91052D922E28A38F
+uid           [ultimate] Sheng Zha (szha) <[email protected]>
+sig 3        91052D922E28A38F 2018-08-23  Sheng Zha (szha) <[email protected]>
+sub   rsa4096 2018-08-23 [E]
+sig          91052D922E28A38F 2018-08-23  Sheng Zha (szha) <[email protected]>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFt+Ey0BEAC1+bteK715FULfFz4ItsgifWAN1GkclMRvhcZxGl6dH6GraOlQ
+b06ctlJk0g7uzvFCXIKvnE8GIN3ZH5zQzS1mXmeLWSicnA9sHNrwZQi6GCKsAuLF
+HZC5WED7yEGRzMbeOc7oov7yROjgWEb+cq7qdcdR9ZfYU0genGLFraxw/buALASW
+ohB0Iu7p0Hf/VQFQZJJOXDUSRJ9Bh9yrVizRGhHRy54Ex3spWCMvTadfui0qDUEq
+c2O+FudCnZFWlP4VYZWWp3sPLBIcIiSLHutgsh7dmJNOSNChzxjwHrtclweaWdUQ
+2PLtuDzeE7GIkuAWCl37w9pjbL+IIVTyN/4exTjhN8bFjbCc38reFKbJR6X5S7OP
+ucozmZb5vqOous7RY5SLBfhseZLG0TJY0n0cl3enolt5sPWBDI/smL5mtCZN06mX
+kDNbSMEBFzWnRUg7aJu041lU7W2KAKIydpKSDB3gQT59BDP8jaGc/UBBZGZNechs
+bN6P/dJ34BONJbBAqWzk7H1PBdH4zgewtttA9ZMqx4GIlQpd8/J9eWGM+W5Be6X+
+uRBBps3jCX1K+BJYOcCijUTUrrApyt7eoPKzznK37NMqNN7mt+Bjc5cN5FXT5ChT
+mJry/dkfw3ROeonk/kWy4FQLEfTUvOVy0vs2cxYPJanv1/l0X9xWw44+eQARAQAB
+tCZTaGVuZyBaaGEgKHN6aGEpIDx6aGFzaGVuZ0BhcGFjaGUub3JnPokCTgQTAQgA
+OBYhBMuRZMdqgD2GHeFuW5EFLZIuKKOPBQJbfhMtAhsDBQsJCAcCBhUKCQgLAgQW
+AgMBAh4BAheAAAoJEJEFLZIuKKOPoK0QAJYmMaLCoDT93xbIVz247Y07Qs9isFJ5
+ZDaOOuJE8dhJbwl9rZ1gD3KUZuJ9hZk5kdHdE6NcV2+eSSdcIHgy6WT0iwLFe8KD
+sd5nVjUBhQNjz4ecNwwAqBRWDnXq0Qf+fFeja2vrDkruQ0yHAAxdzf1fqh2xOuoS
+yn8IKyAIOE2U0QrPo3otHdou5zEzlDuZqWhbn1mHwk22Y/D/AWgTjJCk9Mn4ZxNf
+7kIESUt5ekzf6Woc9du2zpKD0Zt0RElC6xkLBBhnn1BcEio+bzxD3PQwBGVrVfxe
+CLumDM9LqS7e20o9sYQiNbpDtpAV0iYiUL5lvekXEHCE0zaaRCWLjZsDzY2BgADZ
+OaC87tep57Ug9FlSWeU9wtoo/78DXGL8f2M1teUmA1Em1D7Mxql0zNgmaHXBpkxK
+9b45x9C9DvfKbVOVr14IA/jJzxpxsLDjynLc36SguGbiDRcHcL63K2u+leo7kH9X
+a6UpTm+n6JXGgPlhTvznAz8ZQCitEv/8zQHhAVZS7A+lKdLbY/m0niyd9rKMFZnA
+SWUTnJu0U0wQnxdP76otQ8d4UjfzLYANlI+CMyoZPbJDi9Yg2TsU/vqMDuFs9Vj+
+Dkh6wAVIakvrUzgPwy22jmiXA0r5PMhtMoi3YbGXcsjnf+gkThIDasE5X7qCmGnH
+fnDgkTUMZ1mIuQINBFt+Ey0BEAC2BM64GDpmGGwrcPt1v4HFoISQhdv1s67IE0XY
+rwV3jFuyXk0gYPbGnCW9QCphUA8/jes0iZHYRMcd0PI9maUQCYZJUI00qtRwVW8v
+tVCwxh/SrdIo+HiCNsfQ7Y0byeeFVsDyzlg/VG8kjsjQsOEK7iZzvWczDcS1E7NF
+leg/17jrU6Rpg2rJlLWah/aYHgB32naf2niH3kRQYvE3ZizdBXxCw8WCGE9ZPeZt
+kE/yBz8vw1vYdCYtoPXBLO1N6ugjuxdhuOC+QBDcbwzKfO5EouxRIG49GMy4o9zC
+PYwST7rXOoTacMRQm676OeXGSLiiR9ywTgkDPsYJkCI/yf10W2XHHvK7bPcsFbU9
+zifr74WSWLC5UVQCdiEK30nqSrKmDTPHqgkDftXH6d+K+upPh4iNjUBroGZxORWo
+aDmkXBvccUOCwfO9w9UTDg7lSxUZC8Uq13QQUK4oKAOlvdD3gmeORhxsdt+6FOtn
+WXX0VyjUZyY1/oOGscNLs1RHKHKRdFtDCDogd3LpFcU5QTiYcRX0fEQD7i9zUAGz
+5tXTbL6VWO9kwwyjbWsL2KxPGTj0tmbp/JtcS9wxye7KxWA4eJoMlXCNzQPEdUzh
+SmMlh5AAgvb4eBp3yamBuPd4IVANPLbcbsvkDOiY8Ix/tbjYy8smrwxn7gTokZL1
+DQBXSwARAQABiQI2BBgBCAAgFiEEy5Fkx2qAPYYd4W5bkQUtki4oo48FAlt+Ey0C
+GwwACgkQkQUtki4oo4/AZw//Xr9FOVnxzOtM7xKln3YwFUwr7Z4IR+3Oc6UtKfSO
+C6VDyo0CVIHvTDwijsOZOay5OZqY9VjjctoTSDPnh/gJnSeT56I3VtPPiWuwuOqm
+YD8HnwcOb0ZifeZr9GMgrVehJ4t9dMAIo1l5MWPP5/p+KHm5FZYkZk/5AtsCAxh5
+tBEK001fxWo85pEZfZx1zMAfchhKH3VncMCqAi964+/dOIjUlvpBJBDC29GrqW5Y
+0LP0TsoQ766bWKJ261QyGhILZawu5EDdqwE7BBLT1NQFvQTEkJ8FwurgFTQVfLln
+lAa2DDCPpV0omfUZAr8oHHhNLowIzbqXe3q1mac+Yr1l4L8zZl9gXWGbR/6fvwf/
+yUzaUrh285ZsgsIKACyjV4tsS/iKJv+dM2Hu39OtMtoRI9oKFNHb9HkElHj4MSqF
+SBkyOtsGeqivwPlnZMfzLkicBpmyQHYmgOtfcmSuxhs5aCuespzXfZw8Nv2Aergc
+TWhiJbJn7HPY21RJfLwOgo/o76LmdHhwn+PMhsvxox8pMYigLaTj4himBgk/W8bd
+I2j8j2s5+rQ+LMbaRYUPeqP3/cBl59uIPAMigD8+ZHb4yCOVoBVRtG6LJoLHqRzd
+a4LYL628Ksuv1Yxn/Uhb5nDPxU5RKRDeogn07wtaqSG12T9pcIwmExW1RN5kvXc2
+3Os=
+=XL0V
+-----END PGP PUBLIC KEY BLOCK-----

Makefile

@@ -565,7 +565,6 @@ rcpplint:
 	3rdparty/dmlc-core/scripts/lint.py mxnet-rcpp ${LINT_LANG} R-package/src
 
 rpkg:
-	mkdir -p R-package/inst
 	mkdir -p R-package/inst/libs
 	cp src/io/image_recordio.h R-package/src
 	cp -rf lib/libmxnet.so R-package/inst/libs
@@ -579,16 +578,15 @@ rpkg:
 	echo "import(methods)" >> R-package/NAMESPACE
 	R CMD INSTALL R-package
 	Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package')"
-	rm -rf R-package/NAMESPACE
 	Rscript -e "if (!require('roxygen2')||packageVersion('roxygen2')!= '5.0.1'){\
 	devtools::install_version('roxygen2',version='5.0.1',\
 	repo='https://cloud.r-project.org/',quiet=TRUE)}"
 	Rscript -e "require(roxygen2); roxygen2::roxygenise('R-package')"
 	R CMD INSTALL R-package
-	rm -rf R-package/src/image_recordio.h
 
 rpkgtest:
-	Rscript -e "require(testthat);res<-test_dir('R-package/tests/testthat');if(!testthat:::all_passed(res)){stop('Test failures', call. = FALSE)}"
+	Rscript -e 'require(testthat);res<-test_dir("R-package/tests/testthat");if(!testthat:::all_passed(res)){stop("Test failures", call. = FALSE)}'
+	Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file("r-package_coverage.json");writeLines(covr:::to_codecov(res), fileConn);close(fileConn)'
 
 scalaclean:
 	(cd $(ROOTDIR)/scala-package; \