improving network capabilities

An update of NixOS/nixops#922

- Replaced `deployment.libvirtd.networks` option with a submodule to allow not
  only (libvirt) network names, but other networking types as well.
- Domain XML was adjusted accordingly to incorporate the parameters from the new
  `networks` submodule.
- Added the qemu guest agent to guests to allow for out-of-band
  communication (no need for network connectivity) with the hypervisor.
- Guest IP (for provisioning after guest has started) is no longer determined by
  waiting for the guest to get a DHCP lease in the hypervisor libvirt network.
  If the guest has a static IP, it won't ask for a DHCP lease. Also, for bridged
  networking, we probably will not have access to the DHCP server.
- Instead, the address of the first interface is retrieved from libvirt using
  the `VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT` method, which can now be done
  because of the newly added qemu guest agent.

nix/libvirtd.nix

@@ -82,9 +82,11 @@ in
     };
 
     deployment.libvirtd.networks = mkOption {
-      default = [ "default" ];
-      type = types.listOf types.str;
-      description = "Names of libvirt networks to attach the VM to.";
+      type = types.listOf (types.submodule (import ./network-options.nix {
+        inherit lib;
+      }));
+      default = [{ source = "default"; type= "virtual"; }];
+      description = "Describe network interfaces.";
     };
 
     deployment.libvirtd.extraDevicesXML = mkOption {
@@ -142,6 +144,8 @@ in
     services.openssh.extraConfig = "UseDNS no";
 
     deployment.hasFastConnection = true;
+
+    services.qemuGuest.enable = true;
 };
 
 }

nix/network-options.nix

@@ -0,0 +1,23 @@
+{ lib } :
+
+with lib;
+{
+  options = {
+
+    source = mkOption {
+      type = types.str;
+      default = "default";
+      description = ''
+      '';
+    };
+
+    type = mkOption {
+      type = types.enum [ "bridge" "virtual" ];
+      default = "virtual";
+      description = ''
+      '';
+    };
+
+  };
+
+}

nixopsvirtd/backends/libvirtd.py

@@ -18,6 +18,29 @@
 # to prevent libvirt errors from appearing on screen, see
 # https://www.redhat.com/archives/libvirt-users/2017-August/msg00011.html
 
+
+class LibvirtdNetwork:
+
+    INTERFACE_TYPES = {
+        'virtual': 'network',
+        'bridge': 'bridge',
+    }
+
+    def __init__(self, **kwargs):
+        self.type = kwargs['type']
+        self.source = kwargs['source']
+
+    @property
+    def interface_type(self):
+        return self.INTERFACE_TYPES[self.type]
+
+    @classmethod
+    def from_xml(cls, x):
+        type = x.find("attr[@name='type']/string").get("value")
+        source = x.find("attr[@name='source']/string").get("value")
+        return cls(type=type, source=source)
+
+
 class LibvirtdDefinition(MachineDefinition):
     """Definition of a trivial machine."""
 
@@ -35,6 +58,9 @@ def __init__(self, xml, config):
         self.extra_devices = x.find("attr[@name='extraDevicesXML']/string").get("value")
         self.extra_domain = x.find("attr[@name='extraDomainXML']/string").get("value")
         self.headless = x.find("attr[@name='headless']/bool").get("value") == 'true'
+        self.image_dir = x.find("attr[@name='imageDir']/string")
+        if self.image_dir:
+            self.image_dir = self.image_dir.get("value")
         self.domain_type = x.find("attr[@name='domainType']/string").get("value")
         self.kernel = x.find("attr[@name='kernel']/string").get("value")
         self.initrd = x.find("attr[@name='initrd']/string").get("value")
@@ -43,17 +69,21 @@ def __init__(self, xml, config):
         self.uri = x.find("attr[@name='URI']/string").get("value")
 
         self.networks = [
-            k.get("value")
-            for k in x.findall("attr[@name='networks']/list/string")]
+            LibvirtdNetwork.from_xml(n)
+            for n in x.findall("attr[@name='networks']/list/*")
+        ]
+
+        print("%r" % self.networks)
+        # print("%r" % self.networks[0])
+        # print("%r" % self.networks[1])
+        print("len=%d" % len(self.networks))
         assert len(self.networks) > 0
 
 
 class LibvirtdState(MachineState):
     private_ipv4 = nixops.util.attr_property("privateIpv4", None)
     client_public_key = nixops.util.attr_property("libvirtd.clientPublicKey", None)
     client_private_key = nixops.util.attr_property("libvirtd.clientPrivateKey", None)
-    primary_net = nixops.util.attr_property("libvirtd.primaryNet", None)
-    primary_mac = nixops.util.attr_property("libvirtd.primaryMAC", None)
     domain_xml = nixops.util.attr_property("libvirtd.domainXML", None)
     disk_path = nixops.util.attr_property("libvirtd.diskPath", None)
     storage_volume_name = nixops.util.attr_property("libvirtd.storageVolume", None)
@@ -134,17 +164,9 @@ def address_to(self, m):
     def _vm_id(self):
         return "nixops-{0}-{1}".format(self.depl.uuid, self.name)
 
-    def _generate_primary_mac(self):
-        mac = [0x52, 0x54, 0x00,
-               random.randint(0x00, 0x7f),
-               random.randint(0x00, 0xff),
-               random.randint(0x00, 0xff)]
-        self.primary_mac = ':'.join(map(lambda x: "%02x" % x, mac))
-
     def create(self, defn, check, allow_reboot, allow_recreate):
         assert isinstance(defn, LibvirtdDefinition)
         self.set_common_state(defn)
-        self.primary_net = defn.networks[0]
         self.storage_pool_name = defn.storage_pool_name
         self.uri = defn.uri
 
@@ -153,14 +175,13 @@ def create(self, defn, check, allow_reboot, allow_recreate):
         if self.conn.getLibVersion() < 1002007:
             raise Exception('libvirt 1.2.7 or newer is required at the target host')
 
-        if not self.primary_mac:
-            self._generate_primary_mac()
+        self.storage_pool_name = defn.storage_pool_name
 
         if not self.client_public_key:
             (self.client_private_key, self.client_public_key) = nixops.util.create_key_pair()
 
         if self.storage_volume_name is None:
-            self._prepare_storage_volume()
+            self._prepare_storage_volume(defn)
             self.storage_volume_name = self.vol.name()
 
         self.domain_xml = self._make_domain_xml(defn)
@@ -178,7 +199,7 @@ def create(self, defn, check, allow_reboot, allow_recreate):
         self.start()
         return True
 
-    def _prepare_storage_volume(self):
+    def _prepare_storage_volume(self, defn):
         self.logger.log("preparing disk image...")
         newEnv = copy.deepcopy(os.environ)
         newEnv["NIXOPS_LIBVIRTD_PUBKEY"] = self.client_public_key
@@ -196,27 +217,40 @@ def _prepare_storage_volume(self):
 
         self.logger.log("uploading disk image...")
         image_info = self._get_image_info(temp_disk_path)
-        self._vol = self._create_volume(image_info['virtual-size'], image_info['actual-size'])
+        self._vol = self._create_volume(image_info['virtual-size'], image_info['actual-size'], defn.image_dir)
         self._upload_volume(temp_disk_path, image_info['actual-size'])
 
     def _get_image_info(self, filename):
         output = self._logged_exec(["qemu-img", "info", "--output", "json", filename], capture_stdout=True)
         return json.loads(output)
 
-    def _create_volume(self, virtual_size, actual_size):
+    def _create_volume(self, virtual_size, actual_size, path=None):
+        # according to https://libvirt.org/formatstorage.html#StoragePoolTarget
+        # files should be created with rights depending on parent folder but 
+        # this doesn't seem true
+        # here I hardcode permission rights (BAD)
         xml = '''
         <volume>
           <name>{name}</name>
           <capacity>{virtual_size}</capacity>
           <allocation>{actual_size}</allocation>
           <target>
             <format type="qcow2"/>
+            <permissions>
+                <owner>1000</owner>
+                <group>100</group>
+                <mode>0744</mode>
+                <label>virt_image_t</label>
+            </permissions>
+            {eventual_path}
           </target>
         </volume>
         '''.format(
             name="{}.qcow2".format(self._vm_id()),
             virtual_size=virtual_size,
             actual_size=actual_size,
+            # eventual_path= "<path >%s</path>" % path if path else ""
+            eventual_path= ""
         )
         vol = self.pool.createXML(xml)
         self._vol = vol
@@ -243,19 +277,15 @@ def _get_qemu_executable(self):
     def _make_domain_xml(self, defn):
         qemu = self._get_qemu_executable()
 
-        def maybe_mac(n):
-            if n == self.primary_net:
-                return '<mac address="' + self.primary_mac + '" />'
-            else:
-                return ""
-
         def iface(n):
             return "\n".join([
-                '    <interface type="network">',
-                maybe_mac(n),
-                '      <source network="{0}"/>',
+                '    <interface type="{interface_type}">',
+                '      <source {interface_type}="{source}"/>',
                 '    </interface>',
-            ]).format(n)
+            ]).format(
+                interface_type=n.interface_type,
+                source=n.source,
+            )
 
         def _make_os(defn):
             return [
@@ -283,6 +313,10 @@ def _make_os(defn):
             '    <graphics type="vnc" port="-1" autoport="yes"/>' if not defn.headless else "",
             '    <input type="keyboard" bus="usb"/>',
             '    <input type="mouse" bus="usb"/>',
+            '    <channel type="unix">',
+            '      <target type="virtio" name="org.qemu.guest_agent.0"/>',
+            '      <address type="virtio-serial" controller="0" bus="0" port="1"/>',
+            '    </channel>',
             defn.extra_devices,
             '  </devices>',
             defn.extra_domain,
@@ -302,19 +336,39 @@ def _parse_ip(self):
         """
         return an ip v4
         """
-        # alternative is VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE if qemu agent is available
-        ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE, 0)
-        if ifaces is None:
-            self.log("Failed to get domain interfaces")
+
+        try:
+            ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT, 0)
+            # if ifaces is None:
+            #     self.log("Failed to get domain interfaces")
+            #     return
+            # print("The interface IP addresses:")
+            from ipaddress import ip_address
+            for (name, val) in ifaces.iteritems():
+                self.log("Parsing interface %s..." % name)
+
+                if val['addrs']:
+                    for ipaddr in val['addrs']:
+                        curaddr = ip_address(unicode(ipaddr['addr']))
+
+                        if ipaddr['type'] == libvirt.VIR_IP_ADDR_TYPE_IPV4:
+                            print(ipaddr['addr'] + " VIR_IP_ADDR_TYPE_IPV4")
+                            if curaddr.is_loopback:
+                                continue
+                            self.success("Found address")
+                            return ipaddr['addr']
+
+                        else:
+                            pass
+
+        except libvirt.libvirtError as e:
+            self.log(str(e))
             return
 
-        for (name, val) in ifaces.iteritems():
-            if val['addrs']:
-                for ipaddr in val['addrs']:
-                    return ipaddr['addr']
+        return False
+
 
     def _wait_for_ip(self, prev_time):
-        self.log_start("waiting for IP address to appear in DHCP leases...")
         while True:
             ip = self._parse_ip()
             if ip:
@@ -334,7 +388,6 @@ def _is_running(self):
     def start(self):
         assert self.vm_id
         assert self.domain_xml
-        assert self.primary_net
         if self._is_running():
             self.log("connecting...")
             self.private_ipv4 = self._parse_ip()