From 5ed94d14d9e1aadc6cc80ad895ef9eb342fe90e9 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron Date: Thu, 12 Sep 2019 02:24:17 +0900 Subject: [PATCH] improving network capabilities An update of https://github.com/NixOS/nixops/pull/922 - Replaced `deployment.libvirtd.networks` option with a submodule to allow not only (libvirt) network names, but other networking types as well. - Domain XML was adjusted accordingly to incorporate the parameters from the new `networks` submodule. - Added the qemu guest agent to guests to allow for out-of-band communication (no need for network connectivity) with the hypervisor. - Guest IP (for provisioning after guest has started) is no longer determined by waiting for the guest to get a DHCP lease in the hypervisor libvirt network. If the guest has a static IP, it won't ask for a DHCP lease. Also, for bridged networking, we probably will not have access to the DHCP server. - Instead, the address of the first interface is retrieved from libvirt using the `VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT` method, which can now be done because of the newly added qemu guest agent. --- nix/libvirtd.nix | 10 ++- nix/network-options.nix | 23 ++++++ nixopsvirtd/backends/libvirtd.py | 129 ++++++++++++++++++++++--------- 3 files changed, 121 insertions(+), 41 deletions(-) create mode 100644 nix/network-options.nix diff --git a/nix/libvirtd.nix b/nix/libvirtd.nix index e956bd4..d21c4e5 100644 --- a/nix/libvirtd.nix +++ b/nix/libvirtd.nix @@ -82,9 +82,11 @@ in }; deployment.libvirtd.networks = mkOption { - default = [ "default" ]; - type = types.listOf types.str; - description = "Names of libvirt networks to attach the VM to."; + type = types.listOf (types.submodule (import ./network-options.nix { + inherit lib; + })); + default = [{ source = "default"; type= "virtual"; }]; + description = "Describe network interfaces."; }; deployment.libvirtd.extraDevicesXML = mkOption { @@ -142,6 +144,8 @@ in services.openssh.extraConfig = "UseDNS no"; deployment.hasFastConnection = true; + + services.qemuGuest.enable = true; }; } diff --git a/nix/network-options.nix b/nix/network-options.nix new file mode 100644 index 0000000..d72da59 --- /dev/null +++ b/nix/network-options.nix @@ -0,0 +1,23 @@ +{ lib } : + +with lib; +{ + options = { + + source = mkOption { + type = types.str; + default = "default"; + description = '' + ''; + }; + + type = mkOption { + type = types.enum [ "bridge" "virtual" ]; + default = "virtual"; + description = '' + ''; + }; + + }; + +} diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index e383b40..359cb2d 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -18,6 +18,29 @@ # to prevent libvirt errors from appearing on screen, see # https://www.redhat.com/archives/libvirt-users/2017-August/msg00011.html + +class LibvirtdNetwork: + + INTERFACE_TYPES = { + 'virtual': 'network', + 'bridge': 'bridge', + } + + def __init__(self, **kwargs): + self.type = kwargs['type'] + self.source = kwargs['source'] + + @property + def interface_type(self): + return self.INTERFACE_TYPES[self.type] + + @classmethod + def from_xml(cls, x): + type = x.find("attr[@name='type']/string").get("value") + source = x.find("attr[@name='source']/string").get("value") + return cls(type=type, source=source) + + class LibvirtdDefinition(MachineDefinition): """Definition of a trivial machine.""" @@ -35,6 +58,9 @@ def __init__(self, xml, config): self.extra_devices = x.find("attr[@name='extraDevicesXML']/string").get("value") self.extra_domain = x.find("attr[@name='extraDomainXML']/string").get("value") self.headless = x.find("attr[@name='headless']/bool").get("value") == 'true' + self.image_dir = x.find("attr[@name='imageDir']/string") + if self.image_dir: + self.image_dir = self.image_dir.get("value") self.domain_type = x.find("attr[@name='domainType']/string").get("value") self.kernel = x.find("attr[@name='kernel']/string").get("value") self.initrd = x.find("attr[@name='initrd']/string").get("value") @@ -43,8 +69,14 @@ def __init__(self, xml, config): self.uri = x.find("attr[@name='URI']/string").get("value") self.networks = [ - k.get("value") - for k in x.findall("attr[@name='networks']/list/string")] + LibvirtdNetwork.from_xml(n) + for n in x.findall("attr[@name='networks']/list/*") + ] + + print("%r" % self.networks) + # print("%r" % self.networks[0]) + # print("%r" % self.networks[1]) + print("len=%d" % len(self.networks)) assert len(self.networks) > 0 @@ -52,8 +84,6 @@ class LibvirtdState(MachineState): private_ipv4 = nixops.util.attr_property("privateIpv4", None) client_public_key = nixops.util.attr_property("libvirtd.clientPublicKey", None) client_private_key = nixops.util.attr_property("libvirtd.clientPrivateKey", None) - primary_net = nixops.util.attr_property("libvirtd.primaryNet", None) - primary_mac = nixops.util.attr_property("libvirtd.primaryMAC", None) domain_xml = nixops.util.attr_property("libvirtd.domainXML", None) disk_path = nixops.util.attr_property("libvirtd.diskPath", None) storage_volume_name = nixops.util.attr_property("libvirtd.storageVolume", None) @@ -134,17 +164,9 @@ def address_to(self, m): def _vm_id(self): return "nixops-{0}-{1}".format(self.depl.uuid, self.name) - def _generate_primary_mac(self): - mac = [0x52, 0x54, 0x00, - random.randint(0x00, 0x7f), - random.randint(0x00, 0xff), - random.randint(0x00, 0xff)] - self.primary_mac = ':'.join(map(lambda x: "%02x" % x, mac)) - def create(self, defn, check, allow_reboot, allow_recreate): assert isinstance(defn, LibvirtdDefinition) self.set_common_state(defn) - self.primary_net = defn.networks[0] self.storage_pool_name = defn.storage_pool_name self.uri = defn.uri @@ -153,14 +175,13 @@ def create(self, defn, check, allow_reboot, allow_recreate): if self.conn.getLibVersion() < 1002007: raise Exception('libvirt 1.2.7 or newer is required at the target host') - if not self.primary_mac: - self._generate_primary_mac() + self.storage_pool_name = defn.storage_pool_name if not self.client_public_key: (self.client_private_key, self.client_public_key) = nixops.util.create_key_pair() if self.storage_volume_name is None: - self._prepare_storage_volume() + self._prepare_storage_volume(defn) self.storage_volume_name = self.vol.name() self.domain_xml = self._make_domain_xml(defn) @@ -178,7 +199,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.start() return True - def _prepare_storage_volume(self): + def _prepare_storage_volume(self, defn): self.logger.log("preparing disk image...") newEnv = copy.deepcopy(os.environ) newEnv["NIXOPS_LIBVIRTD_PUBKEY"] = self.client_public_key @@ -196,14 +217,18 @@ def _prepare_storage_volume(self): self.logger.log("uploading disk image...") image_info = self._get_image_info(temp_disk_path) - self._vol = self._create_volume(image_info['virtual-size'], image_info['actual-size']) + self._vol = self._create_volume(image_info['virtual-size'], image_info['actual-size'], defn.image_dir) self._upload_volume(temp_disk_path, image_info['actual-size']) def _get_image_info(self, filename): output = self._logged_exec(["qemu-img", "info", "--output", "json", filename], capture_stdout=True) return json.loads(output) - def _create_volume(self, virtual_size, actual_size): + def _create_volume(self, virtual_size, actual_size, path=None): + # according to https://libvirt.org/formatstorage.html#StoragePoolTarget + # files should be created with rights depending on parent folder but + # this doesn't seem true + # here I hardcode permission rights (BAD) xml = ''' {name} @@ -211,12 +236,21 @@ def _create_volume(self, virtual_size, actual_size): {actual_size} + + 1000 + 100 + 0744 + + + {eventual_path} '''.format( name="{}.qcow2".format(self._vm_id()), virtual_size=virtual_size, actual_size=actual_size, + # eventual_path= "%s" % path if path else "" + eventual_path= "" ) vol = self.pool.createXML(xml) self._vol = vol @@ -243,19 +277,15 @@ def _get_qemu_executable(self): def _make_domain_xml(self, defn): qemu = self._get_qemu_executable() - def maybe_mac(n): - if n == self.primary_net: - return '' - else: - return "" - def iface(n): return "\n".join([ - ' ', - maybe_mac(n), - ' ', + ' ', + ' ', ' ', - ]).format(n) + ]).format( + interface_type=n.interface_type, + source=n.source, + ) def _make_os(defn): return [ @@ -283,6 +313,10 @@ def _make_os(defn): ' ' if not defn.headless else "", ' ', ' ', + ' ', + ' ', + '
', + ' ', defn.extra_devices, ' ', defn.extra_domain, @@ -302,19 +336,39 @@ def _parse_ip(self): """ return an ip v4 """ - # alternative is VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE if qemu agent is available - ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE, 0) - if ifaces is None: - self.log("Failed to get domain interfaces") + + try: + ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT, 0) + # if ifaces is None: + # self.log("Failed to get domain interfaces") + # return + # print("The interface IP addresses:") + from ipaddress import ip_address + for (name, val) in ifaces.iteritems(): + self.log("Parsing interface %s..." % name) + + if val['addrs']: + for ipaddr in val['addrs']: + curaddr = ip_address(unicode(ipaddr['addr'])) + + if ipaddr['type'] == libvirt.VIR_IP_ADDR_TYPE_IPV4: + print(ipaddr['addr'] + " VIR_IP_ADDR_TYPE_IPV4") + if curaddr.is_loopback: + continue + self.success("Found address") + return ipaddr['addr'] + + else: + pass + + except libvirt.libvirtError as e: + self.log(str(e)) return - for (name, val) in ifaces.iteritems(): - if val['addrs']: - for ipaddr in val['addrs']: - return ipaddr['addr'] + return False + def _wait_for_ip(self, prev_time): - self.log_start("waiting for IP address to appear in DHCP leases...") while True: ip = self._parse_ip() if ip: @@ -334,7 +388,6 @@ def _is_running(self): def start(self): assert self.vm_id assert self.domain_xml - assert self.primary_net if self._is_running(): self.log("connecting...") self.private_ipv4 = self._parse_ip()