Add helper to get hosts on metricbeat integration tests (#11925)

Add a helper to automatically obtain the host of a service in metricbeat
integration tests so tests don't depend on environment variables or
current running docker services. This is needed to allow in the future
to run multiple versions of the same service, is also helpful when
running tests locally.

Environment variables with the format `SERVICE_HOST`, being SERVICE
the uppercased name of the compose service can be used to override the
automatic value.

`NO_COMPOSE` environment variable can be used now to avoid stopping
the running containers too.

Add ports to all services in docker compose files, required for the
helper.

Part of #7957 for python-based integration tests.

libbeat/tests/system/beat/compose.py

@@ -1,10 +1,14 @@
 from __future__ import absolute_import
 import os
+import sys
 import time
 
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 
+# TODO: To be removed when env files are removed from docker compose
+OVERRIDE_HOST = os.environ.get('OVERRIDE_HOST', False)
+
 
 if INTEGRATION_TESTS:
     from compose.cli.command import get_project
@@ -89,21 +93,67 @@ def compose_down(cls):
         """
         Stop all running containers
         """
+        if os.environ.get('NO_COMPOSE'):
+            return
+
         if INTEGRATION_TESTS and cls.COMPOSE_SERVICES:
             cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
 
     @classmethod
-    def compose_hosts(cls):
+    def get_hosts(cls):
+        return [cls.compose_host()]
+
+    @classmethod
+    def _private_host(cls, info, port):
+        """
+        Return the address of the container, it should be reachable from the
+        host if docker is being run natively. To be used when the tests are
+        run from another container in the same network. It also works when
+        running from the host network if the docker daemon runs natively.
+        """
+        networks = info['NetworkSettings']['Networks'].values()
+        port = port.split("/")[0]
+        for network in networks:
+            ip = network['IPAddress']
+            if ip:
+                return "%s:%s" % (ip, port)
+
+    @classmethod
+    def _exposed_host(cls, info, port):
+        """
+        Return the exposed address in the host, can be used when the test is
+        run from the host network. Recommended when using docker machines.
+        """
+        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
+        return "localhost:%s" % hostPort
+
+    @classmethod
+    def compose_host(cls, service=None, port=None):
         if not INTEGRATION_TESTS or not cls.COMPOSE_SERVICES:
             return []
 
-        hosts = []
-        for container in cls.compose_project().containers(service_names=cls.COMPOSE_SERVICES):
-            network_settings = container.inspect()['NetworkSettings']
-            for network in network_settings['Networks'].values():
-                if network['IPAddress']:
-                    hosts.append(network['IPAddress'])
-        return hosts
+        if service is None:
+            service = cls.COMPOSE_SERVICES[0]
+
+        # TODO: Remove condition when env files are removed from docker compose
+        if OVERRIDE_HOST:
+            host_env = os.environ.get(service.upper() + "_HOST")
+            if host_env:
+                return host_env
+
+        container = cls.compose_project().containers(service_names=[service])[0]
+        info = container.inspect()
+        portsConfig = info['HostConfig']['PortBindings']
+        if len(portsConfig) == 0:
+            raise Exception("No exposed ports for service %s" % service)
+        if port is None:
+            port = portsConfig.keys()[0]
+
+        # We can use _exposed_host for all platforms when we can use host network
+        # in the metricbeat container
+        if sys.platform.startswith('linux'):
+            return cls._private_host(info, port)
+        return cls._exposed_host(info, port)
 
     @classmethod
     def compose_project(cls):