Synchronise yoga with upstream

.zuul.yaml

@@ -197,24 +197,11 @@
     parent: devstack-tempest
     description: |
       Run tempest compute API tests using LVM image backend. This only runs
-      against nova/virt/libvirt/* changes.
-    # Copy irrelevant-files from nova-dsvm-multinode-base and then exclude
-    # anything that is not in nova/virt/libvirt/* or nova/privsep/*.
-    irrelevant-files:
-      - ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/privsep/).*$
-      - ^api-.*$
-      - ^(test-|)requirements.txt$
-      - ^.*\.rst$
-      - ^.git.*$
-      - ^doc/.*$
-      - ^nova/hacking/.*$
-      - ^nova/locale/.*$
-      - ^nova/tests/.*$
-      - ^nova/test.py$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
+      against nova/virt/libvirt/*, nova/privsep/* and .zuul.yaml changes.
+    files:
+      - ^nova/virt/libvirt/.*$
+      - ^nova/privsep/.*$
+      - .zuul.yaml
     vars:
       # We use the "all" environment for tempest_test_regex and
       # tempest_exclude_regex.
@@ -253,22 +240,11 @@
     # NOTE(chateaulav): due to constraints with no IDE support for aarch64,
     # tests have been limited to eliminate any items that are incompatible.
     # This is to be re-evaluated as greater support is added and defined.
-    irrelevant-files:
-      - ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/objects/)(?!nova/scheduler/).*$
-      - ^api-.*$
-      - ^(test-|)requirements.txt$
-      - ^.*\.rst$
-      - ^.git.*$
-      - ^doc/.*$
-      - ^nova/hacking/.*$
-      - ^nova/locale/.*$
-      - ^nova/policies/.*$
-      - ^nova/tests/.*$
-      - ^nova/test.py$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
+    files:
+      - ^nova/virt/libvirt/.*$
+      - ^nova/objects/.*$
+      - ^nova/scheduler/.*$
+      - .zuul.yaml
     vars:
       tox_envlist: all
       tempest_test_regex: ^tempest\.(api\.compute\.servers|scenario\.test_network_basic_ops)
@@ -580,6 +556,16 @@
         GLANCE_STANDALONE: True
         GLANCE_USE_IMPORT_WORKFLOW: True
         DEVSTACK_PARALLEL: True
+        MYSQL_REDUCE_MEMORY: True
+      # NOTE(danms): This job is pretty heavy as it is, so we disable some
+      # services that are not relevant to the nova-glance-ceph scenario
+      # that this job is intended to validate.
+      devstack_services:
+        c-bak: false
+        s-account: false
+        s-container: false
+        s-object: false
+        s-proxy: false
       devstack_local_conf:
         post-config:
           $NOVA_CONF:
@@ -632,11 +618,12 @@
         - nova-ceph-multistore:
             irrelevant-files: *nova-base-irrelevant-files
         - neutron-linuxbridge-tempest:
-            irrelevant-files:
+            files:
               # NOTE(mriedem): This job has its own irrelevant-files section
               # so that we only run it on changes to networking and libvirt/vif
               # code; we don't need to run this on all changes.
-              - ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$
+              - ^nova/network/.*$
+              - nova/virt/libvirt/vif.py
         - nova-live-migration
         - nova-live-migration-ceph
         - nova-lvm
@@ -680,11 +667,6 @@
         - barbican-tempest-plugin-simple-crypto:
             irrelevant-files: *nova-base-irrelevant-files
             voting: false
-        - tempest-integrated-compute-centos-8-stream:
-            irrelevant-files: *nova-base-irrelevant-files
-        - tempest-centos8-stream-fips:
-            irrelevant-files: *nova-base-irrelevant-files
-            voting: false
     gate:
       jobs:
         - nova-live-migration
@@ -697,11 +679,12 @@
         - nova-ceph-multistore:
             irrelevant-files: *nova-base-irrelevant-files
         - neutron-linuxbridge-tempest:
-            irrelevant-files:
+            files:
               # NOTE(mriedem): This job has its own irrelevant-files section
               # so that we only run it on changes to networking and libvirt/vif
               # code; we don't need to run this on all changes.
-              - ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$
+              - ^nova/network/.*$
+              - nova/virt/libvirt/vif.py
         - tempest-integrated-compute:
            irrelevant-files: *policies-irrelevant-files
         - nova-grenade-multinode:
@@ -710,8 +693,6 @@
             irrelevant-files: *nova-base-irrelevant-files
         - openstacksdk-functional-devstack:
             irrelevant-files: *nova-base-irrelevant-files
-        - tempest-integrated-compute-centos-8-stream:
-            irrelevant-files: *nova-base-irrelevant-files
     experimental:
       jobs:
         - ironic-tempest-bfv:

nova/compute/manager.py

@@ -6646,9 +6646,9 @@ def _shelve_offload_instance(self, context, instance, clean_shutdown,
 
         instance.power_state = current_power_state
         # NOTE(mriedem): The vm_state has to be set before updating the
-        # resource tracker, see vm_states.ALLOW_RESOURCE_REMOVAL. The host/node
-        # values cannot be nulled out until after updating the resource tracker
-        # though.
+        # resource tracker, see vm_states.allow_resource_removal(). The
+        # host/node values cannot be nulled out until after updating the
+        # resource tracker though.
         instance.vm_state = vm_states.SHELVED_OFFLOADED
         instance.task_state = None
         instance.save(expected_task_state=[task_states.SHELVING,

nova/compute/resource_tracker.py

@@ -1495,7 +1495,8 @@ def _update_usage_from_instance(self, context, instance, nodename,
         # NOTE(sfinucan): Both brand new instances as well as instances that
         # are being unshelved will have is_new_instance == True
         is_removed_instance = not is_new_instance and (is_removed or
-            instance['vm_state'] in vm_states.ALLOW_RESOURCE_REMOVAL)
+            vm_states.allow_resource_removal(
+                vm_state=instance['vm_state'], task_state=instance.task_state))
 
         if is_new_instance:
             self.tracked_instances.add(uuid)
@@ -1554,7 +1555,9 @@ def _update_usage_from_instances(self, context, instances, nodename):
 
         instance_by_uuid = {}
         for instance in instances:
-            if instance.vm_state not in vm_states.ALLOW_RESOURCE_REMOVAL:
+            if not vm_states.allow_resource_removal(
+                    vm_state=instance['vm_state'],
+                    task_state=instance.task_state):
                 self._update_usage_from_instance(context, instance, nodename)
             instance_by_uuid[instance.uuid] = instance
         return instance_by_uuid

nova/compute/stats.py

@@ -105,7 +105,8 @@ def update_stats_for_instance(self, instance, is_removed=False):
         (vm_state, task_state, os_type, project_id) = \
                 self._extract_state_from_instance(instance)
 
-        if is_removed or vm_state in vm_states.ALLOW_RESOURCE_REMOVAL:
+        if is_removed or vm_states.allow_resource_removal(
+                vm_state=vm_state, task_state=task_state):
             self._decrement("num_instances")
             self.states.pop(uuid)
         else:

nova/compute/vm_states.py

@@ -27,6 +27,7 @@
 See http://wiki.openstack.org/VMState
 """
 
+from nova.compute import task_states
 from nova.objects import fields
 
 
@@ -74,5 +75,11 @@
 # states we allow to trigger crash dump
 ALLOW_TRIGGER_CRASH_DUMP = [ACTIVE, PAUSED, RESCUED, RESIZED, ERROR]
 
-# states we allow resources to be freed in
-ALLOW_RESOURCE_REMOVAL = [DELETED, SHELVED_OFFLOADED]
+
+def allow_resource_removal(vm_state, task_state=None):
+    """(vm_state, task_state) combinations we allow resources to be freed in"""
+
+    return (
+        vm_state == DELETED or
+        vm_state == SHELVED_OFFLOADED and task_state != task_states.SPAWNING
+    )

nova/conf/workarounds.py

@@ -416,6 +416,16 @@
         help="""
 When this is enabled, it will skip version-checking of hypervisors
 during live migration.
+"""),
+    cfg.BoolOpt(
+        'disable_deep_image_inspection',
+        default=False,
+        help="""
+This disables the additional deep image inspection that the compute node does
+when downloading from glance. This includes backing-file, data-file, and
+known-features detection *before* passing the image to qemu-img. Generally,
+this inspection should be enabled for maximum safety, but this workaround
+option allows disabling it if there is a compatibility concern.
 """),
 ]