From bfb097a3486256d5112d92b3c398a70814e9fc74 Mon Sep 17 00:00:00 2001 From: Titouan Rigoudy Date: Fri, 30 Apr 2021 16:15:42 +0200 Subject: [PATCH] Add ability to run servers on different IP addresses. --- tools/serve/serve.py | 98 ++++++++++++++++--------------- tools/wptserve/wptserve/config.py | 61 ++++++++++++++++++- 2 files changed, 109 insertions(+), 50 deletions(-) diff --git a/tools/serve/serve.py b/tools/serve/serve.py index 096851d07c546b3..52a1287c4a8e696 100644 --- a/tools/serve/serve.py +++ b/tools/serve/serve.py @@ -507,7 +507,7 @@ def start(self, init_func, host, port, paths, routes, bind_address, config, **kw self.proc = self.mp_context.Process(target=self.create_daemon, args=(init_func, host, port, paths, routes, bind_address, config), - name='%s on port %s' % (self.scheme, port), + name='%s on %s:%s' % (self.scheme, host, port), kwargs=kwargs) self.proc.daemon = True self.proc.start() @@ -588,24 +588,30 @@ def check_subdomains(config, routes, mp_context): "on {}. {}".format(url, EDIT_HOSTS_HELP)) sys.exit(1) - for domain in config.domains_set: - if domain == host: + for host_key, subdomains in config.domains.items(): + if host_key in config.alternate_ip_addresses: + # These domains are served on a different IP than `host`. continue - try: - urllib.request.urlopen("http://%s:%d/" % (domain, port)) - except Exception: - logger.critical("Failed probing domain {}. {}".format(domain, EDIT_HOSTS_HELP)) - sys.exit(1) + for subdomain, domain in subdomains.items(): + if domain == host: + # Already tested above. + continue + + try: + urllib.request.urlopen("http://%s:%d/" % (domain, port)) + except Exception: + logger.critical("Failed probing domain {}. {}".format(domain, EDIT_HOSTS_HELP)) + sys.exit(1) wrapper.wait() -def make_hosts_file(config, host): +def make_hosts_file(config, default_ip_address): rv = [] - for domain in config.domains_set: - rv.append("%s\t%s\n" % (host, domain)) + for domain, ip_address in config.domains_ip_addresses.items(): + rv.append("%s\t%s\n" % (ip_address or default_ip_address, domain)) # Windows interpets the IP address 0.0.0.0 as non-existent, making it an # appropriate alias for non-existent hosts. However, UNIX-like systems @@ -621,32 +627,35 @@ def make_hosts_file(config, host): return "".join(rv) -def start_servers(host, ports, paths, routes, bind_address, config, - mp_context, **kwargs): +def start(config, routes, mp_context, **kwargs): servers = defaultdict(list) - for scheme, ports in ports.items(): - assert len(ports) == {"http": 2, "https": 2}.get(scheme, 1) - # If trying to start HTTP/2.0 server, check compatibility - if scheme == 'h2' and not http2_compatible(): - logger.error('Cannot start HTTP/2.0 server as the environment is not compatible. ' + - 'Requires Python 2.7.10+ or 3.6+ and OpenSSL 1.0.2+') - continue + logger.debug("Using ports: %r" % config.ports) + + for host in config.all_server_hosts: + for scheme, ports in config.ports.items(): + assert len(ports) == {"http": 2, "https": 2}.get(scheme, 1) - for port in ports: - if port is None: + # If trying to start HTTP/2.0 server, check compatibility + if scheme == 'h2' and not http2_compatible(): + logger.error('Cannot start HTTP/2.0 server as the environment is not compatible. ' + + 'Requires Python 2.7.10+ or 3.6+ and OpenSSL 1.0.2+') continue - init_func = {"http": start_http_server, - "https": start_https_server, - "h2": start_http2_server, - "ws": start_ws_server, - "wss": start_wss_server, - "quic-transport": start_quic_transport_server}[scheme] - - server_proc = ServerProc(mp_context, scheme=scheme) - server_proc.start(init_func, host, port, paths, routes, bind_address, - config, **kwargs) - servers[scheme].append((port, server_proc)) + + for port in ports: + if port is None: + continue + init_func = {"http": start_http_server, + "https": start_https_server, + "h2": start_http2_server, + "ws": start_ws_server, + "wss": start_wss_server, + "quic-transport": start_quic_transport_server}[scheme] + + server_proc = ServerProc(mp_context, scheme=scheme) + server_proc.start(init_func, host, port, config.paths, routes, config.bind_address, + config, **kwargs) + servers[scheme].append((port, server_proc)) return servers @@ -873,19 +882,6 @@ def start_quic_transport_server(host, port, paths, routes, bind_address, config, startup_failed(log=False) -def start(config, routes, mp_context, **kwargs): - host = config["server_host"] - ports = config.ports - paths = config.paths - bind_address = config["bind_address"] - - logger.debug("Using ports: %r" % ports) - - servers = start_servers(host, ports, paths, routes, bind_address, config, mp_context, **kwargs) - - return servers - - def iter_procs(servers): for servers in servers.values(): for port, server in servers: @@ -927,7 +923,15 @@ class ConfigBuilder(config.ConfigBuilder): _default = { "browser_host": "web-platform.test", "alternate_hosts": { - "alt": "not-web-platform.test" + "alt": "not-web-platform.test", + "private": "private-web-platform.test", + "public": "public-web-platform.test", + }, + "alternate_ip_addresses": { + # Missing hosts use the default IP address passed to + # `make_hosts_file()`. + "private": "127.1.0.1", + "public": "127.2.0.1", }, "doc_root": repo_root, "ws_doc_root": os.path.join(repo_root, "websockets", "handlers"), diff --git a/tools/wptserve/wptserve/config.py b/tools/wptserve/wptserve/config.py index 843f6e664e345c1..84f552a43e531ab 100644 --- a/tools/wptserve/wptserve/config.py +++ b/tools/wptserve/wptserve/config.py @@ -139,6 +139,7 @@ class ConfigBuilder(object): _default = { "browser_host": "localhost", "alternate_hosts": {}, + "alternate_ip_addresses": {}, "doc_root": os.path.dirname("__file__"), "server_host": None, "ports": {"http": [8000]}, @@ -172,10 +173,12 @@ class ConfigBuilder(object): computed_properties = ["log_level", "paths", "server_host", + "all_server_hosts", "ports", "domains", "not_domains", "all_domains", + "domains_ip_addresses", "domains_set", "not_domains_set", "all_domains_set", @@ -280,9 +283,22 @@ def _get_log_level(self, data): def _get_paths(self, data): return {"doc_root": data["doc_root"]} + # Returns the main hostname servers should bind to. def _get_server_host(self, data): return data["server_host"] if data.get("server_host") is not None else data["browser_host"] + # Returns the list of different hostnames servers should bind to. + def _get_all_server_hosts(self, data): + if data["alternate_ip_addresses"]: + assert data["bind_address"], ( + "`bind_address` must be set to `true` in the configuration " + + "when using alternate IP addresses.") + + rv = [data["server_host"]] + for name in data["alternate_ip_addresses"].keys(): + rv.append(data["alternate_hosts"][name]) + return rv + def _get_ports(self, data): new_ports = defaultdict(list) for scheme, ports in data["ports"].items(): @@ -293,6 +309,36 @@ def _get_ports(self, data): new_ports[scheme].append(real_port) return new_ports + # Returns a two-level dict: host_key -> subdomain -> full_domain_name + # + # Where: + # + # - `host_key` is either "" for `server_host` or one of the keys in + # `alternate_hosts` + # - `subdomain_key` is one of the values in `subdomains`, or "". + # + # For example, given: + # + # server_host = "foo.example" + # alternate_hosts = {"bar": "bar.example"} + # subdomains = ["hello", "你好"] + # + # This returns: + # + # { + # "": { + # "" : "foo.example", + # "hello": "hello.foo.example", + # "你好" : "xn--6qq79v.foo.example", + # }, + # "bar": { + # "" : "bar.example", + # "hello": "hello.bar.example", + # "你好" : "xn--6qq79v.bar.example", + # }, + # } + # + # Note: the leaves of this tree are all the subdomains in `domains_set`. def _get_domains(self, data): hosts = data["alternate_hosts"].copy() assert "" not in hosts @@ -323,10 +369,19 @@ def _get_all_domains(self, data): rv[host].update(nd[host]) return rv + # Returns a dict mapping each domain to the IP address it should be served + # on. Domains that should use the default IP address map to `None`. + def _get_domains_ip_addresses(self, data): + rv = {} + for host, subdomains in data["domains"].items(): + ip_address = data["alternate_ip_addresses"].get(host, None) + for subdomain in subdomains.values(): + rv[subdomain] = ip_address + return rv + + # The set of all domains / leaves of the `domains` tree. def _get_domains_set(self, data): - return {domain - for per_host_domains in data["domains"].values() - for domain in per_host_domains.values()} + return {domain for domain in data["domains_ip_addresses"].keys()} def _get_not_domains_set(self, data): return {domain