tests: allow test cases to use in-tree data files

Similar to angband/angband#5518 , but test cases can use one of two sets of paths with init_file_paths():  the paths used by main.c or the path set by the preprocessor macro, TEST_DEFAULT_PATH.  The latter is the default, but the user has the option to force the former via a command-line argument, -f, to the test case or run-tests or by setting an environment variable, FORCE_PATH, to a non-empty value.  That's to allow for testing against the installed version of the data files without recompiling the test cases.

Unlike angband/angband#5518 , builds using configure or CMake use a relative path to the in-tree data files:  they set TEST_OVERRIDE_PATHS which triggers TEST_DEFAULT_PATH to be "." + PATH_SEP + "lib" + PATH_SEP in unit-test.h.  That simplifies getting test cases to work in environments like MSYS2 on Windows (paths constructed in the build system are Unix-style and need to be processed through cygpath to get a DOS-style path appropriate for hardwiring into an executable).  Running the test cases through the build system (i.e. "make tests" when using configure) handles setting the working directory so the relative path works.  With configure and a build done from the top-level of the distribution, directly running run-tests or a test case from the top-level directory of the distribution will work as expected (i.e. "src/tests/run-tests" or "src/tests/effects/chain.exe").

Like angband/angband#5518 , adds a check target to Makefile as a synonym for the tests target.

CMakeLists.txt

@@ -744,16 +744,22 @@ FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-test
 
 # Set up target for running all tests.  Those (from tests in the source tree)
 # using the test front end will be run directly.  The unit tests will be
-# handled through a dependency.  Note that when SHARED_INSTALL or
-# READONLY_INSTALL is set, the tests can only be successfully run after
-# installing.  In the SHARED_INSTALL case, some unit tests will fail even
-# after installation is done because the test case executables are not setgid.
+# handled through a dependency.  In the SHARED_INSTALL case, some unit tests
+# will always fail because the test case executables are not setgid.
 ADD_CUSTOM_TARGET(alltests
     COMMAND ${CMAKE_CURRENT_BINARY_DIR}/run-tests ${TEST_EXECUTABLE}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+# For unit tests, use a working directory so the data files can be found.
+# This complements how DEFAULT_CONFIG_PATH, DEFAULT_LIB_PATH, DEFAULT_DATA_PATH,
+# and TEST_OVERRIDE_PATHS are set below.
+IF((READONLY_INSTALL) OR (SHARED_INSTALL))
+    SET(TEST_WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+ELSE()
+    SET(TEST_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+ENDIF()
 ADD_CUSTOM_TARGET(allunittests
-    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/unittests/run-tests
-    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+    COMMAND "${CMAKE_CURRENT_BINARY_DIR}/unittests/run-tests"
+    WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}")
 IF(SUPPORT_TEST_FRONTEND)
     ADD_DEPENDENCIES(alltests OurExecutable)
     IF(SC_INSTALL)
@@ -781,6 +787,13 @@ TARGET_COMPILE_DEFINITIONS(OurUnitTestLib PRIVATE "${ANGBAND_BUILD_ID_OPTION}")
 TARGET_COMPILE_DEFINITIONS(OurUnitTestLib PRIVATE -D DEFAULT_CONFIG_PATH="${ANGBAND_CONFIG_PATH}")
 TARGET_COMPILE_DEFINITIONS(OurUnitTestLib PRIVATE -D DEFAULT_LIB_PATH="${ANGBAND_LIB_PATH}")
 TARGET_COMPILE_DEFINITIONS(OurUnitTestLib PRIVATE -D DEFAULT_DATA_PATH="${ANGBAND_DATA_PATH}")
+# If building for installation elsewhere allows those paths to be overridden
+# so the test cases can be run before installation.  To run the cases after
+# installation with the installed data files, set the environment variable
+# FORCE_PATH to be non-empty prior to running the tests.
+IF((READONLY_INSTALL) OR (SHARED_INSTALL))
+    TARGET_COMPILE_DEFINITIONS(OurUnitTestLib PRIVATE -D TEST_OVERRIDE_PATHS)
+ENDIF()
 
 # Set up targets for exercising the unit tests (from src/tests in the source
 # tree).
@@ -833,5 +846,13 @@ FOREACH(ANGBAND_TEST_CASE ${ANGBAND_TEST_CASE_LIST})
         TARGET_LINK_LIBRARIES(${ANGBAND_TEST_CASE_NAME} PRIVATE ${MATH_LIBRARY})
     ENDIF()
     ADD_DEPENDENCIES(allunittests ${ANGBAND_TEST_CASE_NAME})
+    # Add a shortcut for running (and compiling if necessary) this test case.
+    ADD_CUSTOM_TARGET(run-${ANGBAND_TEST_CASE_NAME}
+        COMMAND "${CMAKE_CURRENT_BINARY_DIR}/${ANGBAND_TEST_CASE_PATH}"
+        WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}")
+    ADD_DEPENDENCIES(run-${ANGBAND_TEST_CASE_NAME} ${ANGBAND_TEST_CASE_NAME})
+    IF(SC_INSTALL)
+        ADD_DEPENDENCIES(run-${ANGBAND_TEST_CASE_NAME} TransferLib)
+    ENDIF()
     MATH(EXPR ANGBAND_TEST_CASE_INDEX "${ANGBAND_TEST_CASE_INDEX} + 1")
 ENDFOREACH()

Makefile

@@ -7,7 +7,8 @@ DISTCLEAN = config.status config.log docs/.deps \
 	mk/buildsys.mk mk/extra.mk
 REPOCLEAN = aclocal.m4 autom4te.cache configure src/autoconf.h.in version
 
-.PHONY: tests manual manual-optional dist
+.PHONY: check tests manual manual-optional dist
+check: tests
 tests:
 	$(MAKE) -C src tests
 

configure.ac

@@ -168,29 +168,41 @@ if test "x$bindir" = "x\${exec_prefix}/bin"; then
   bindir=${exec_prefix}/games
 fi
 
-if test "x$with_no_install" != "x" || test "x$enable_win" = xyes ; then
+if test "x$with_no_install" != "x"; then
 	configpath="${PWD}/lib/"
+elif test "x$enable_win" = xyes; then
+	configpath=".\\\\lib\\\\"
 else
 	configpath="${sysconfdir}/${PACKAGE}/"
 fi
 
-case "/$configpath" in
-	*/) MY_EXPAND_DIR(configdir, "$configpath")  ;;
-	*)  MY_EXPAND_DIR(configdir, "$configpath/") ;;
-esac
+if test "x$enable_win" != xyes; then
+	case "/$configpath" in
+		*/) MY_EXPAND_DIR(configdir, "$configpath")  ;;
+		*)  MY_EXPAND_DIR(configdir, "$configpath/") ;;
+	esac
+else
+	configdir="$configpath"
+fi
 
 
-if test "x$with_no_install" != "x" || test "x$enable_win" = xyes ; then
+if test "x$with_no_install" != "x"; then
 	libpath="${PWD}/lib/"
 	bindir=".."
+elif test "x$enable_win" = xyes; then
+	libpath=".\\\\lib\\\\"
 else
 	libpath="${datarootdir}/${PACKAGE}/"
 fi
 
-case "/$libpath" in
-	*/) MY_EXPAND_DIR(libdatadir, "$libpath")  ;;
-	*)  MY_EXPAND_DIR(libdatadir, "$libpath/") ;;
-esac
+if test "x$enable_win" != xyes; then
+	case "/$libpath" in
+		*/) MY_EXPAND_DIR(libdatadir, "$libpath")  ;;
+		*)  MY_EXPAND_DIR(libdatadir, "$libpath/") ;;
+	esac
+else
+	libdatadir="$libpath"
+fi
 
 if test "x$with_no_install" != "x"; then
 	docdir="${PWD}/doc/"
@@ -206,16 +218,23 @@ case "/$docdir" in
 	*)  MY_EXPAND_DIR(docdatadir, "$docdir/") ;;
 esac
 
-if test "x$with_no_install" != "x" || test "x$enable_win" = xyes ; then
+if test "x$with_no_install" != "x"; then
 	varpath="${PWD}/lib/"
+elif test "x$enable_win" = xyes; then
+	varpath=".\\\\lib\\\\"
 else
 	varpath="${localstatedir}/games/${PACKAGE}/"
 fi
 
-case "/$varpath" in
-	*/) MY_EXPAND_DIR(vardatadir, "$varpath")  ;;
-	*)  MY_EXPAND_DIR(vardatadir, "$varpath/") ;;
-esac
+if test "x$enable_win" != xyes; then
+	case "/$varpath" in
+		*/) MY_EXPAND_DIR(vardatadir, "$varpath")  ;;
+		*)  MY_EXPAND_DIR(vardatadir, "$varpath/") ;;
+	esac
+else
+	vardatadir="$varpath"
+fi
+
 if test "x$with_private_dirs" != "x"; then
 	varshareddatadir="${vardatadir}user/"
 else
@@ -533,6 +552,25 @@ else
 fi
 AC_SUBST(CROSS_COMPILE)
 
+dnl If installing elsewhere or using the Windows front end, allow the test
+dnl cases which use set_file_paths() to find the data files.  This relies on
+dnl the setting for TEST_DEFAULT_PATH in src/tests/unit-test.h which uses a
+dnl relative path and has the drawback that a test case will not work if it
+dnl uses set_file_paths() and the working directory is not the top level
+dnl directory of a distribution.  Also set TEST_WORKING_DIRECTORY so, that
+dnl when run by src/tests/run-tests, the working directory will match that.
+if test x"$with_no_install" = x || test x"$enable_win" = xyes ; then
+	CFLAGS="${CFLAGS} -DTEST_OVERRIDE_PATHS"
+	case "$srcdir" in
+		.) TEST_WORKING_DIRECTORY="$PWD" ;;
+		[\\/]* | ?:[\\/]* ) TEST_WORKING_DIRECTORY="$srcdir" ;;
+		*) TEST_WORKING_DIRECTORY="$PWD/$srcdir" ;;
+	esac
+else
+	TEST_WORKING_DIRECTORY=
+fi
+AC_SUBST(TEST_WORKING_DIRECTORY)
+
 AC_SUBST(MAINFILES, ${MAINFILES})
 AC_SUBST(TEST_LIBS, ${TEST_LIBS})
 AC_CONFIG_FILES([mk/buildsys.mk mk/extra.mk])

docs/hacking/compiling.rst

@@ -74,20 +74,22 @@ To compile and run the unit tests, do this::
     make -f Makefile.osx tests
 
 If you want to rerun just one part, say monster/attack, of the unit tests,
-that's most easily done by::
+that's most easily done from the top directory of the source distribution::
+
+    src/tests/monster/attack.exe
+
+Somewhat older versions configure the test cases so they should be run
+from src/tests.  For those you would either use::
 
     cd src/tests
     monster/attack.exe
 
-Previous versions put the test executables in src/tests/bin.  With those
-versions, the second line above would be::
+or, for the even older versions that put the test executables in src/tests/bin,
+use::
 
+    cd src/tests
     bin/monster/attack
 
-The reason for changing directories to src/tests is to match up with how the
-tests were compiled:  they expect Angband's configuration data to be in
-../../lib.
-
 Statistics build
 ~~~~~~~~~~~~~~~~
 

mk/extra.mk.in

@@ -58,6 +58,7 @@ prefix ?= @prefix@
 VERSION ?= @VERSION@
 MAINFILES = @MAINFILES@
 TEST_LIBS = @TEST_LIBS@
+TEST_WORKING_DIRECTORY ?= @TEST_WORKING_DIRECTORY@
 USE_STATS = @USE_STATS@
 SPHINXBUILD ?= @SPHINXBUILD@
 DOC_HTML_THEME ?= @DOC_HTML_THEME@

src/Makefile

@@ -35,10 +35,13 @@ $(PROGNAME).o: $(OBJECTS)
 	$(LD) $(LDFLAGS) -nostdlib -r -o $@ $(OBJECTS)
 	@printf "%10s %-20s\n" LINK $@
 
+check: tests
 tests: $(PROGNAME).o
 	env CC="$(CC)" CFLAGS="$(CFLAGS)" CPPFLAGS="$(CPPFLAGS)" \
 		LDFLAGS="$(LDFLAGS)" LDADD="$(LDADD)" LIBS="$(TEST_LIBS)" \
-		CROSS_COMPILE="$(CROSS_COMPILE)" $(MAKE) -C tests all
+		CROSS_COMPILE="$(CROSS_COMPILE)" \
+		TEST_WORKING_DIRECTORY="$(TEST_WORKING_DIRECTORY)" \
+		$(MAKE) -C tests all
 
 test-depgen:
 	env CC="$(CC)" $(MAKE) -C tests depgen
@@ -128,4 +131,4 @@ post-install:
 	fi
 
 FORCE :
-.PHONY : tests coverage clean-coverage tests/ran-already
+.PHONY : check tests coverage clean-coverage tests/ran-already

src/Makefile.osx

@@ -220,13 +220,17 @@ dist: install
 
 	rm -rf disttemp*
 
+check: tests
 tests: $(EXE).o
-	$(MAKE) -C tests all CFLAGS='-I.. -DDEFAULT_CONFIG_PATH=\"../../lib\" -DDEFAULT_LIB_PATH=\"../../lib\" -DDEFAULT_DATA_PATH=\"../../lib\" $(CFLAGS) -Wno-write-strings' LDFLAGS="$(LIBS)"
+	env CC="$(CC)" CFLAGS="$(CFLAGS) -I.. -DTEST_OVERRIDE_PATHS -Wno-write-strings" \
+		LDFLAGS="$(LDFLAGS)" LIBS="$(LIBS)" \
+		TEST_WORKING_DIRECTORY="../.." \
+		$(MAKE) -C tests all
 
 test-depgen:
 	$(MAKE) -C tests depgen
 
 test-clean:
 	$(MAKE) -C tests clean
 
-.PHONY : tests test-clean
+.PHONY : check tests test-clean

src/tests/run-tests

@@ -4,25 +4,29 @@
 use warnings FATAL => 'all';
 use strict;
 use File::Basename qw(dirname basename);
+use File::Spec::Functions qw(rel2abs);
 use List::Util qw(first max);
 use Getopt::Long qw(:config bundling no_ignore_case);
 
 # some nice global variables
-my $quiet    = 0;
-my $verbose  = $ENV{VERBOSE};
-my $usecolor = 1;
+my $quiet     = 0;
+my $verbose   = $ENV{VERBOSE};
+my $forcepath = $ENV{FORCE_PATH};
+my $usecolor  = 1;
 
 sub usage {
     my $prog = basename($0);
     print <<USAGE;
 Usage: $prog [options]
 
 Options:
-    -h,--help        show this message
-    -c,--color       use ANSI colors (default)
-    -C,--no-color    don't use ANSI colors
-    -q,--quiet       only show summary output
-    -v,--verbose     show all test output
+    -h,--help          show this message
+    -c,--color         use ANSI colors (default)
+    -C,--no-color      don't use ANSI colors
+    -q,--quiet         only show summary output
+    -v,--verbose       show all test output
+    -f,--forcepath     force test cases to use the game's data file paths
+    -F,--no-forcepath  test cases use alternate data file paths (default)
 
 Runs all the unit tests and reports the results.
 USAGE
@@ -44,27 +48,37 @@ sub getcolor {
 
 sub main {
     GetOptions(
-        'help|h'     => sub { usage(0) },
-        'color|c'    => sub { $usecolor = 1 },
-        'no-color|C' => sub { $usecolor = 0 },
-        'verbose|v'  => sub { $verbose = 1; $quiet = 0 },
-        'quiet|q'    => sub { $quiet = 1; $verbose = 0 },
+        'help|h'         => sub { usage(0) },
+        'color|c'        => sub { $usecolor = 1 },
+        'no-color|C'     => sub { $usecolor = 0 },
+        'verbose|v'      => sub { $verbose = 1; $quiet = 0 },
+        'quiet|q'        => sub { $quiet = 1; $verbose = 0 },
+        'forcepath|f'    => sub { $forcepath = 1 },
+        'no-forcepath|F' => sub { $forcepath = 0 },
     ) || usage(1);
 
-    my $dir     = dirname($0);
+    # Want the absolute path so that changing directories before running the
+    # test does not invalidate the results from find.
+    my $dir     = rel2abs(dirname($0));
     my @paths   = `find $dir -mindepth 2 -maxdepth 2 -type f -perm -u+x`;
     my $pass    = 0;
     my $total   = 0;
     my $maxpath = (max map { length($_) } @paths) - 3;
-    my $len     = $maxpath + 1 + 7;
+    my $len     = $maxpath - length($dir) + 3 + 7;
+    my $workdir = $ENV{TEST_WORKING_DIRECTORY};
     my $exitcode = 0;
 
+    if (length($workdir)) {
+        chdir $workdir;
+    }
     print "Running ", scalar(@paths), " suites:\n" unless $quiet;
     foreach my $path (sort @paths) {
         chomp $path;
 
         # actually run the test program here, getting the lines of output
-        my @lines = $verbose ? `$path -v` : `$path`;
+        my @lines = ($verbose and $forcepath) ? `$path -vf` :
+                    $verbose ? `$path -v` :
+                    $forcepath ? `$path -f` : `$path`;
 
         if ($? != 0) {
             print red("$path: Suite died"), "\n";

src/tests/test-utils.c

@@ -13,6 +13,7 @@
 #include "mon-make.h"
 #include "mon-util.h"
 #include "test-utils.h"
+#include "unit-test.h"
 #include "z-util.h"
 
 #if defined(SOUND_SDL) || defined(SOUND_SDL2)
@@ -43,9 +44,24 @@ errr init_sound_win(struct sound_hooks *hooks, int argc, char **argv)
 void set_file_paths(void) {
 	char configpath[512], libpath[512], datapath[512];
 
+	/*
+	 * Allow TEST_DEFAULT_PATH to set all the paths for init_file_paths()
+	 * if it is set and the user has not requested that the default paths
+	 * be used.  TEST_DEFAULT_PATH would typically point to the top level
+	 * of a source distribution + PATH_SEP + lib.  Could use a relative
+	 * path, in which case it should be set so that it works from the
+	 * working directory when a test case is run.
+	 */
 	my_strcpy(configpath, DEFAULT_CONFIG_PATH, sizeof(configpath));
 	my_strcpy(libpath, DEFAULT_LIB_PATH, sizeof(libpath));
 	my_strcpy(datapath, DEFAULT_DATA_PATH, sizeof(datapath));
+#ifdef TEST_DEFAULT_PATH
+	if (!forcepath) {
+		my_strcpy(configpath, TEST_DEFAULT_PATH, sizeof(configpath));
+		my_strcpy(libpath, TEST_DEFAULT_PATH, sizeof(libpath));
+		my_strcpy(datapath, TEST_DEFAULT_PATH, sizeof(datapath));
+	}
+#endif /* TEST_DEFAULT_PATH */
 
 	configpath[511] = libpath[511] = datapath[511] = '\0';