added basic testing for docker

.gitignore

@@ -8,3 +8,7 @@ build/
 dist/
 *egg-info/
 *.swp
+
+Dockerfile.*
+Singularity.*
+test-reports/

easybuild/main.py

@@ -389,6 +389,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     if try_to_generate and build_specs and not generated_ecs:
         easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
 
+    # create a container
+    if options.containerize:
+        _log.info("Creating %s container" % options.container_type)
+        containerize(easyconfigs, options.container_type)
+        cleanup(logfile, eb_tmpdir, testing)
+        sys.exit(0)
+
     forced = options.force or options.rebuild
     dry_run_mode = options.dry_run or options.dry_run_short
 
@@ -415,13 +422,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
         ordered_ecs = []
 
-    # create a container
-    if options.containerize:
-        _log.info("Creating %s container" % options.container_type)
-        containerize(ordered_ecs, eb_go)
-        cleanup(logfile, eb_tmpdir, testing)
-        sys.exit(0)
-
     # creating/updating PRs
     if new_update_preview_pr:
         if options.new_pr:

easybuild/tools/containers/__init__.py

@@ -1,6 +1,5 @@
 from vsc.utils import fancylogger
 
-from easybuild.tools.config import build_option
 from easybuild.tools.config import CONT_TYPE_SINGULARITY, CONT_TYPE_DOCKER
 from easybuild.tools.build_log import EasyBuildError
 from .singularity import singularity as singularity_containerize
@@ -9,16 +8,15 @@
 _log = fancylogger.getLogger('tools.containers')  # pylint: disable=C0103
 
 
-def containerize(easyconfigs, eb_go=None):
+def containerize(easyconfigs, container_type):
     """
     Generate container recipe + (optionally) image
     """
     _log.experimental("support for generating container recipes and images (--containerize/-C)")
 
-    container_type = build_option('container_type')
     if container_type == CONT_TYPE_SINGULARITY:
         singularity_containerize(easyconfigs)
     elif container_type == CONT_TYPE_DOCKER:
-        docker_containerize(easyconfigs, eb_go)
+        docker_containerize(easyconfigs)
     else:
         raise EasyBuildError("Unknown container type specified: %s", container_type)

easybuild/tools/containers/docker.py

@@ -107,14 +107,14 @@ def check_docker_containerize():
     docker_path = which('docker')
     if not docker_path:
         raise EasyBuildError("docker executable not found.")
-    _log.debug("found docker executable '%s'" % docker_path)
+    print_msg("docker tool found at %s" % docker_path)
 
     sudo_path = which('sudo')
     if not sudo_path:
         raise EasyBuildError("sudo not found.")
 
     try:
-        run_cmd("sudo docker --version")
+        run_cmd("sudo docker --version", trace=False, force_in_dry_run=True)
     except Exception:
         raise EasyBuildError("Error getting docker version")
 
@@ -130,7 +130,7 @@ def _det_os_deps(easyconfigs):
     return res
 
 
-def generate_dockerfile(easyconfigs, container_base, eb_go):
+def generate_dockerfile(easyconfigs, container_base):
     os_deps = _det_os_deps(easyconfigs)
 
     module_naming_scheme = ActiveMNS()
@@ -141,9 +141,7 @@ def generate_dockerfile(easyconfigs, container_base, eb_go):
 
     mod_names = [e['ec'].full_mod_name for e in easyconfigs]
 
-    eb_opts = [eb_opt for eb_opt in eb_go.generate_cmd_line()
-               if not eb_opt.startswith('--container') and eb_opt not in ['--ignore-osdeps', '--experimental']]
-    eb_opts.extend(eb_go.args)
+    eb_opts = [os.path.basename(ec['spec']) for ec in easyconfigs]
 
     tmpl = _DOCKER_TMPLS[container_base]
     content = tmpl % {
@@ -159,7 +157,7 @@ def generate_dockerfile(easyconfigs, container_base, eb_go):
     if img_name:
         file_label = os.path.splitext(img_name)[0]
     else:
-        file_label = mod_names[-1].replace('/', '-')
+        file_label = mod_names[0].replace('/', '-')
 
     dockerfile = os.path.join(cont_path, 'Dockerfile.%s' % file_label)
     if os.path.exists(dockerfile):
@@ -169,6 +167,7 @@ def generate_dockerfile(easyconfigs, container_base, eb_go):
             raise EasyBuildError("Dockerfile at %s already exists, not overwriting it without --force", dockerfile)
 
     write_file(dockerfile, content)
+    print_msg("Dockerfile file created at %s" % dockerfile, log=_log)
 
     return dockerfile
 
@@ -180,20 +179,23 @@ def build_docker_image(easyconfigs, dockerfile):
     module_name = module_naming_scheme.det_full_module_name(ec)
 
     tempdir = tempfile.mkdtemp(prefix='easybuild-docker')
-    container_name = build_option('container_image_name') or "%s:latest" % module_name
+    container_name = build_option('container_image_name') or "%s:latest" % module_name.replace('/', '-')
     docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.'])
-    run_cmd(docker_cmd, path=tempdir)
+
+    print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd)
+    run_cmd(docker_cmd, path=tempdir, stream_output=True)
+    print_msg("Docker image created at %s" % container_name, log=_log)
 
     shutil.rmtree(tempdir)
 
 
-def docker_containerize(easyconfigs, eb_go):
+def docker_containerize(easyconfigs):
 
     check_docker_containerize()
 
     # Generate dockerfile
     container_base = build_option('container_base') or DEFAULT_DOCKER_BASE_IMAGE
-    dockerfile = generate_dockerfile(easyconfigs, container_base, eb_go)
+    dockerfile = generate_dockerfile(easyconfigs, container_base)
 
     # Build image if requested
     if build_option('container_build_image'):

test/framework/containers.py

@@ -59,6 +59,12 @@
 fi
 """
 
+MOCKED_DOCKER = """\
+echo "docker was called with arguments: $@"
+echo "$@"
+echo $#
+"""
+
 
 class ContainersTest(EnhancedTestCase):
     """Tests for containers support"""
@@ -89,11 +95,11 @@ def test_parse_container_base(self):
             expected.update({'arg2': 'bar'})
             self.assertEqual(parse_container_base('%s:foo:bar' % agent), expected)
 
-    def run_main(self, args):
+    def run_main(self, args, raise_error=False):
         """Helper function to run main with arguments specified in 'args' and return stdout/stderr."""
         self.mock_stdout(True)
         self.mock_stderr(True)
-        self.eb_main(args, raise_error=True, verbose=True, do_build=True)
+        self.eb_main(args, raise_error=raise_error, verbose=True, do_build=True)
         stdout = self.get_stdout().strip()
         stderr = self.get_stderr().strip()
         self.mock_stdout(False)
@@ -120,6 +126,7 @@ def test_end2end_singularity_recipe(self):
         args = [
             toy_ec,
             '--containerize',
+            '--container-type=singularity',
             '--experimental',
         ]
 
@@ -138,7 +145,7 @@ def test_end2end_singularity_recipe(self):
             remove_file(os.path.join(self.test_prefix, 'containers', 'Singularity.toy-0.0'))
 
         args.append("--container-base=shub:test123")
-        self.run_main(args)
+        self.run_main(args, raise_error=True)
 
         # existing definition file is not overwritten without use of --force
         error_pattern = "Container recipe at .* already exists, not overwriting it without --force"
@@ -204,6 +211,7 @@ def test_end2end_singularity_image(self):
             toy_ec,
             '-C',  # equivalent with --containerize
             '--experimental',
+            '--container-type=singularity',
             '--container-base=localimage:%s' % test_img,
             '--container-build-image',
         ]
@@ -277,6 +285,106 @@ def test_end2end_singularity_image(self):
         self.assertFalse(stderr)
         self.check_regexs(regexs, stdout)
 
+    def test_end2end_dockerfile(self):
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+        containerpath = os.path.join(self.test_prefix, 'containers')
+        os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
+        # --containerpath must be an existing directory (this is done to avoid misconfiguration)
+        mkdir(containerpath)
+
+        base_args = [
+            toy_ec,
+            '--containerize',
+            '--container-type=docker',
+            '--experimental',
+        ]
+
+        error_pattern = "Unsupported container base image 'not-supported'"
+        self.assertErrorRegex(EasyBuildError,
+                              error_pattern,
+                              self.eb_main,
+                              base_args + ['--container-base=not-supported'],
+                              raise_error=True)
+
+        for cont_base in ['ubuntu:16.04', 'centos:7']:
+            stdout, stderr = self.run_main(base_args + ['--container-base=%s' % cont_base])
+            self.assertFalse(stderr)
+            regexs = ["^== Dockerfile file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix]
+            self.check_regexs(regexs, stdout)
+            remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
+
+        self.run_main(base_args + ['--container-base=centos:7'], raise_error=True)
+
+        error_pattern = "Dockerfile at .* already exists, not overwriting it without --force"
+        self.assertErrorRegex(EasyBuildError,
+                              error_pattern,
+                              self.eb_main,
+                              base_args + ['--container-base=centos:7'],
+                              raise_error=True)
+
+        remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
+
+        base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb'))
+        self.run_main(base_args, raise_error=True)
+        def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
+        regexs = [
+            "FROM ubuntu:16.04",
+            "eb toy-0.0.eb GCC-4.9.2.eb",
+            "module load toy/0.0 GCC/4.9.2",
+        ]
+        self.check_regexs(regexs, def_file)
+        remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
+
+    def test_end2end_docker_image(self):
+
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+        containerpath = os.path.join(self.test_prefix, 'containers')
+        os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
+        # --containerpath must be an existing directory (this is done to avoid misconfiguration)
+        mkdir(containerpath)
+
+        args = [
+            toy_ec,
+            '-C',  # equivalent with --containerize
+            '--experimental',
+            '--container-type=docker',
+            '--container-build-image',
+        ]
+
+        if not which('docker'):
+            error_pattern = "docker executable not found."
+            self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
+
+        # install mocked versions of 'sudo' and 'docker' commands
+        docker = os.path.join(self.test_prefix, 'bin', 'docker')
+        write_file(docker, MOCKED_DOCKER)
+        adjust_permissions(docker, stat.S_IXUSR, add=True)
+
+        sudo = os.path.join(self.test_prefix, 'bin', 'sudo')
+        write_file(sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n')
+        adjust_permissions(sudo, stat.S_IXUSR, add=True)
+
+        os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'bin'), os.getenv('PATH'))
+
+        stdout, stderr = self.run_main(args)
+        self.assertFalse(stderr)
+        regexs = [
+            "^== docker tool found at %s/bin/docker" % self.test_prefix,
+            "^== Dockerfile file created at %s/containers/Dockerfile\.toy-0.0" % self.test_prefix,
+            "^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...",
+            "^== Docker image created at toy-0.0:latest",
+        ]
+        self.check_regexs(regexs, stdout)
+
+        args.extend(['--force', '--extended-dry-run'])
+        stdout, stderr = self.run_main(args)
+        self.assertFalse(stderr)
+        self.check_regexs(regexs, stdout)
+
 
 def suite():
     """ returns all the testcases in this module """