UCL Scotch support for SWIFT

INSTALL.swift

@@ -150,6 +150,15 @@ before you can build it.
         least "--with-metis". ParMETIS is preferred over METIS when there
         is a choice.
 
+- Scotch:
+   a build of the Scotch library should be used to
+        optimize the load between MPI nodes by mapping the decomposed domain 
+        to the available compute. This should be found in the
+        standard installation directories, or pointed at using the
+        "--with-scotch" configuration options.
+        In this case the top-level installation directory of the build
+        should be given. 
+
 - libNUMA:
 	a build of the NUMA library can be used to pin the threads to
         the physical core of the machine SWIFT is running on. This is

Makefile.am

@@ -48,8 +48,8 @@ EXTRA_LIBS = $(GSL_LIBS) $(HDF5_LIBS) $(FFTW_LIBS) $(NUMA_LIBS) $(PROFILER_LIBS)
 	$(CHEALPIX_LIBS)
 
 # MPI libraries.
-MPI_LIBS = $(PARMETIS_LIBS) $(METIS_LIBS) $(MPI_THREAD_LIBS) $(FFTW_MPI_LIBS)
-MPI_FLAGS = -DWITH_MPI $(PARMETIS_INCS) $(METIS_INCS) $(FFTW_MPI_INCS)
+MPI_LIBS = $(SCOTCH_LIBS) $(PARMETIS_LIBS) $(METIS_LIBS) $(MPI_THREAD_LIBS) $(FFTW_MPI_LIBS)
+MPI_FLAGS = -DWITH_MPI $(SCOTCH_INCS) $(PARMETIS_INCS) $(METIS_INCS) $(FFTW_MPI_INCS)
 
 # Programs.
 bin_PROGRAMS = swift

configure.ac

@@ -791,6 +791,41 @@ AC_CHECK_LIB(pthread, posix_fallocate,
 	     AC_DEFINE([HAVE_POSIX_FALLOCATE], [1], [The posix library implements file allocation functions.]),
 	     AC_MSG_WARN(POSIX implementation does not have file allocation functions.))
 
+# Check for SCOTCH.
+have_scotch="no"
+AC_ARG_WITH([scotch],
+    [AS_HELP_STRING([--with-scotch=PATH],
+       [root directory where SCOTCH is installed @<:@yes/no@:>@]
+    )],
+    [with_scotch="$withval"],
+    [with_scotch="no"]
+)
+
+SCOTCH_LIBS=""
+if test "x$with_scotch" != "xno"; then
+
+# Check if we have SCOTCH.
+   if test "x$with_scotch" != "xyes" -a "x$with_scotch" != "x"; then
+      SCOTCH_LIBS="-L$with_scotch/lib -lscotch -lscotcherr"
+      SCOTCH_INCS="-I$with_scotch/include"
+   else
+      SCOTCH_LIBS="-lscotch -lscotcherr"
+      SCOTCH_INCS=""
+   fi
+   AC_CHECK_LIB([scotch],[SCOTCH_graphInit], [have_scotch="yes"],
+                [have_scotch="no"], $SCOTCH_LIBS)
+   if test "$have_scotch" = "yes"; then
+      AC_DEFINE([HAVE_SCOTCH],1,[The SCOTCH library is present.])
+   else
+      AC_MSG_ERROR("Failed to find a SCOTCH library")
+   fi
+fi
+
+AC_SUBST([SCOTCH_LIBS])
+AC_SUBST([SCOTCH_INCS])
+AM_CONDITIONAL([HAVESCOTCH],[test -n "$SCOTCH_LIBS"])
+
+
 # Check for METIS.
 have_metis="no"
 AC_ARG_WITH([metis],
@@ -3051,6 +3086,7 @@ AC_MSG_RESULT([
    HDF5 enabled         : $with_hdf5
     - parallel          : $have_parallel_hdf5
    METIS/ParMETIS       : $have_metis / $have_parmetis
+   Scotch               : $have_scotch
    FFTW3 enabled        : $have_fftw   
     - threaded/openmp   : $have_threaded_fftw / $have_openmp_fftw 
     - MPI               : $have_mpi_fftw

src/Makefile.am

@@ -28,8 +28,8 @@ GIT_CMD = @GIT_CMD@
 EXTRA_LIBS = $(GSL_LIBS) $(HDF5_LIBS) $(FFTW_LIBS) $(NUMA_LIBS) $(PROFILER_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(GRACKLE_LIBS)  $(SUNDIALS_LIBS) $(CHEALPIX_LIBS)
 
 # MPI libraries.
-MPI_LIBS = $(PARMETIS_LIBS) $(METIS_LIBS) $(MPI_THREAD_LIBS)
-MPI_FLAGS = -DWITH_MPI $(PARMETIS_INCS) $(METIS_INCS)
+MPI_LIBS = $(SCOTCH_LIBS) $(PARMETIS_LIBS) $(METIS_LIBS) $(MPI_THREAD_LIBS)
+MPI_FLAGS = -DWITH_MPI $(SCOTCH_INCS) $(PARMETIS_INCS) $(METIS_INCS)
 
 # Build the libswiftsim library and a convenience library just for the gravity tasks
 lib_LTLIBRARIES = libswiftsim.la

src/debug.c

@@ -637,7 +637,8 @@ void dumpCells(const char *prefix, int super, int active, int mpiactive,
   fclose(file);
 }
 
-#if defined(WITH_MPI) && (defined(HAVE_METIS) || defined(HAVE_PARMETIS))
+#if defined(WITH_MPI) && (defined(HAVE_METIS) || defined(HAVE_PARMETIS)) \
+    && !defined(HAVE_SCOTCH)
 
 /**
  * @brief Dump a graph in METIS standard format, simple format and weights
@@ -650,8 +651,7 @@ void dumpCells(const char *prefix, int super, int active, int mpiactive,
  *
  * The output filenames are generated from the prefix and the sequence number
  * of calls. So the first is called {prefix}_std_001.dat,
- *{prefix}_simple_001.dat,
- * {prefix}_weights_001.dat, etc.
+ * {prefix}_simple_001.dat, {prefix}_weights_001.dat, etc.
  *
  * @param prefix base output filename
  * @param nvertices the number of vertices

src/debug.h

@@ -39,8 +39,9 @@ int checkCellhdxmax(const struct cell *c, int *depth);
 void dumpCells(const char *prefix, int super, int active, int mpiactive,
                int pactive, struct space *s, int rank, int step);
 
-#if defined(WITH_MPI) && (defined(HAVE_METIS) || defined(HAVE_PARMETIS))
-#include "metis.h"
+#if defined(WITH_MPI) && (defined(HAVE_METIS) || defined(HAVE_PARMETIS)) \
+    && !defined(HAVE_SCOTCH)
+#include <metis.h>
 void dumpMETISGraph(const char *prefix, idx_t nvtxs, idx_t ncon, idx_t *xadj,
                     idx_t *adjncy, idx_t *vwgt, idx_t *vsize, idx_t *adjwgt);
 #endif

src/engine.c

@@ -182,7 +182,8 @@ void engine_addlink(struct engine *e, struct link **l, struct task *t) {
  */
 void engine_repartition(struct engine *e) {
 
-#if defined(WITH_MPI) && (defined(HAVE_PARMETIS) || defined(HAVE_METIS))
+#if defined(WITH_MPI) && \
+    (defined(HAVE_PARMETIS) || defined(HAVE_METIS) || defined(HAVE_SCOTCH))
 
   ticks tic = getticks();
 
@@ -254,7 +255,7 @@ void engine_repartition(struct engine *e) {
             clocks_getunit());
 #else
   if (e->reparttype->type != REPART_NONE)
-    error("SWIFT was not compiled with MPI and METIS or ParMETIS support.");
+    error("SWIFT was not compiled with MPI and METIS, ParMETIS or SCOTCH support.");
 
   /* Clear the repartition flag. */
   e->forcerepart = 0;