Merge pull request #229 from AlexanderZagaynov/BZ-1493041

Azure labels and tag mapping support for a new refresh

app/models/manageiq/providers/azure/inventory/parser/cloud_manager.rb

@@ -86,6 +86,9 @@ def instances
 
       instance_hardware(persister_instance, instance, series)
       instance_operating_system(persister_instance, instance)
+
+      vm_and_template_labels(persister_instance, instance['tags'] || [])
+      vm_and_template_taggings(persister_instance, map_labels('VmAzure', instance['tags'] || []))
     end
   end
 
@@ -199,6 +202,36 @@ def add_instance_disk(persister_hardware, instance, disk)
     )
   end
 
+  def vm_and_template_labels(resource, tags)
+    tags.each do |tag|
+      persister
+        .vm_and_template_labels
+        .find_or_build_by(
+          :resource => resource,
+          :name     => tag.first,
+        )
+        .assign_attributes(
+          :section => 'labels',
+          :source  => 'azure',
+          :value   => tag.second,
+        )
+    end
+  end
+
+  # Returns array of InventoryObject<Tag>.
+  def map_labels(model_name, labels)
+    label_hashes = labels.collect do |tag|
+      { :name => tag.first, :value => tag.second }
+    end
+    persister.tag_mapper.map_labels(model_name, label_hashes)
+  end
+
+  def vm_and_template_taggings(resource, tags_inventory_objects)
+    tags_inventory_objects.each do |tag|
+      persister.vm_and_template_taggings.build(:taggable => resource, :tag => tag)
+    end
+  end
+
   def stacks
     collector.stacks.each do |deployment|
       name = deployment.name

app/models/manageiq/providers/azure/inventory/persister.rb

@@ -24,6 +24,14 @@ def initialize(manager, target = nil, collector = nil)
     initialize_inventory_collections
   end
 
+  def tag_mapper
+    unless defined? @tag_mapper
+      @tag_mapper = ContainerLabelTagMapping.mapper
+      collections[:tags_to_resolve] = @tag_mapper.tags_to_resolve_collection
+    end
+    @tag_mapper
+  end
+
   protected
 
   def cloud

app/models/manageiq/providers/azure/inventory/persister/cloud_manager.rb

@@ -3,7 +3,7 @@ def initialize_inventory_collections
     add_inventory_collections(
       cloud,
       %i(vms miq_templates hardwares operating_systems networks disks availability_zones
-         resource_groups
+         resource_groups vm_and_template_labels vm_and_template_taggings
          flavors key_pairs orchestration_stacks orchestration_stacks_resources
          orchestration_stacks_outputs orchestration_stacks_parameters orchestration_templates)
     )

app/models/manageiq/providers/azure/inventory/persister/target_collection.rb

@@ -16,7 +16,7 @@ def initialize_inventory_collections
     # Child models with references in the Parent InventoryCollections for Cloud
     add_inventory_collections(
       cloud,
-      %i(hardwares operating_systems networks disks
+      %i(hardwares operating_systems networks disks vm_and_template_labels vm_and_template_taggings
          orchestration_stacks_resources orchestration_stacks_outputs orchestration_stacks_parameters)
     )
 

app/models/manageiq/providers/azure/inventory_collection_default/cloud_manager.rb

@@ -60,6 +60,51 @@ def key_pairs(extra_attributes = {})
       super(attributes.merge!(extra_attributes))
     end
 
+    def vm_and_template_labels(extra_attributes = {})
+      attributes = {
+        :model_class                  => CustomAttribute,
+        :association                  => :vm_and_template_labels,
+        :manager_ref                  => %i(resource name),
+        :parent_inventory_collections => %i(vms miq_templates),
+        :inventory_object_attributes  => %i(
+          resource
+          section
+          name
+          value
+          source
+        )
+      }
+
+      attributes[:targeted_arel] = lambda do |inventory_collection|
+        manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).map(&:to_a).flatten
+        inventory_collection.parent.vm_and_template_labels.where(
+          'vms' => {:ems_ref => manager_uuids}
+        )
+      end
+
+      attributes.merge!(extra_attributes)
+    end
+
+    def vm_and_template_taggings(extra_attributes = {})
+      attributes = {
+        :model_class                  => Tagging,
+        :association                  => :vm_and_template_taggings,
+        :manager_ref                  => %i(taggable tag),
+        :inventory_object_attributes  => %i(taggable tag),
+        :parent_inventory_collections => %i(vms miq_templates),
+      }
+
+      attributes[:targeted_arel] = lambda do |inventory_collection|
+        manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).map(&:to_a).flatten
+        ems = inventory_collection.parent
+        ems.vm_and_template_taggings.where(
+          'taggable_id' => ems.vms_and_templates.where(:ems_ref => manager_uuids)
+        )
+      end
+
+      attributes.merge!(extra_attributes)
+    end
+
     def orchestration_stacks(extra_attributes = {})
       attributes = {
         :model_class    => ::ManageIQ::Providers::Azure::CloudManager::OrchestrationStack,

spec/models/manageiq/providers/azure/cloud_manager/azure_refresher_spec_common.rb

@@ -76,6 +76,30 @@ def setup_ems_and_cassette(refresh_settings)
     @ems.reload
   end
 
+  def define_shared_variables
+    _guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
+
+    @ems = FactoryGirl.create(:ems_azure_with_vcr_authentication, :zone => zone, :provider_region => 'eastus')
+
+    @resource_group    = 'miq-azure-test1'
+    @managed_vm        = 'miqazure-linux-managed'
+    @device_name       = 'miq-test-rhel1' # Make sure this is running if generating a new cassette.
+    @vm_powered_off    = 'miqazure-centos1' # Make sure this is powered off if generating a new cassette.
+    @ip_address        = '52.224.165.15'  # This will change if you had to restart the @device_name.
+    @mismatch_ip       = '13.92.63.10'    # This will change if you had to restart the 'miqmismatch1' VM.
+    @managed_os_disk   = "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd"
+    @managed_data_disk = "miqazure-linux-managed-data-disk"
+    @template          = nil
+    @avail_zone        = nil
+
+    @resource_group_managed_vm = "miq-azure-test4"
+
+    FactoryGirl.create(:tag_mapping_with_category,
+                       :labeled_resource_type => 'VmAzure',
+                       :label_name            => 'Shutdown',
+                       :category_name         => 'azure:vm:shutdown')
+  end
+
   def stub_with_current_settings(current_settings)
     stub_settings_merge(
       :ems_refresh => {
@@ -394,7 +418,7 @@ def assert_specific_cloud_network
     expect(vm_subnet.floating_ips.size).to be >= 1
   end
 
-  def assert_specific_vm_powered_on(is_old_refresh = false)
+  def assert_specific_vm_powered_on
     vm = ManageIQ::Providers::Azure::CloudManager::Vm.where(
       :name => @device_name, :raw_power_state => "VM running"
     ).first
@@ -429,14 +453,12 @@ def assert_specific_vm_powered_on(is_old_refresh = false)
     expect(vm.availability_zone).to eql(@avail_zone)
     expect(vm.flavor).to eql(@flavor)
     expect(vm.operating_system.product_name).to eql("RHEL 7.2")
-    expect(vm.custom_attributes.size).to eql(is_old_refresh ? 1 : 0)
+    expect(vm.custom_attributes.size).to eql(1)
     expect(vm.snapshots.size).to eql(0)
 
-    if is_old_refresh
-      aggregate_failures do
-        expect(vm.labels.pluck(:name, :value).to_h).to eq('Shutdown' => 'true')
-        expect(vm.tags.pluck(:name)).to eq(%w(/managed/azure:vm:shutdown/true))
-      end
+    aggregate_failures do
+      expect(vm.labels.pluck(:name, :value).to_h).to eq('Shutdown' => 'true')
+      expect(vm.tags.pluck(:name)).to eq(%w(/managed/azure:vm:shutdown/true))
     end
 
     assert_specific_vm_powered_on_hardware(vm)
@@ -531,7 +553,7 @@ def assert_specific_resource_group
     expect(vm_mismatch.resource_group).to eql(mismatch_group)
   end
 
-  def assert_specific_vm_powered_off(is_old_refresh = false)
+  def assert_specific_vm_powered_off
     vm_name = 'miqazure-centos1'
 
     v = ManageIQ::Providers::Azure::CloudManager::Vm.where(
@@ -552,14 +574,12 @@ def assert_specific_vm_powered_off(is_old_refresh = false)
     expect(v.cloud_network).to eql(cloud_network)
     expect(v.cloud_subnet).to eql(cloud_subnet)
     expect(v.operating_system.product_name).to eql('CentOS 7.1')
-    expect(v.custom_attributes.size).to eql(is_old_refresh ? 1 : 0)
+    expect(v.custom_attributes.size).to eql(1)
     expect(v.snapshots.size).to eql(0)
 
-    if is_old_refresh
-      aggregate_failures do
-        expect(v.labels.pluck(:name, :value).to_h).to eq('Shutdown' => 'true')
-        expect(v.tags.pluck(:name)).to eq(%w(/managed/azure:vm:shutdown/true))
-      end
+    aggregate_failures do
+      expect(v.labels.pluck(:name, :value).to_h).to eq('Shutdown' => 'true')
+      expect(v.tags.pluck(:name)).to eq(%w(/managed/azure:vm:shutdown/true))
     end
 
     assert_specific_vm_powered_off_hardware(v)

spec/models/manageiq/providers/azure/cloud_manager/compare_legacy_and_new_refresh_spec.rb

@@ -7,19 +7,8 @@
   AzureRefresherSpecCommon::ALL_REFRESH_SETTINGS.each do |refresh_settings|
     context "with settings #{refresh_settings}" do
       before do
-        _guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
-
-        @ems = FactoryGirl.create(:ems_azure_with_vcr_authentication, :zone => zone, :provider_region => 'eastus')
-
-        @resource_group    = 'miq-azure-test1'
-        @managed_vm        = 'miqazure-linux-managed'
-        @device_name       = 'miq-test-rhel1' # Make sure this is running if generating a new cassette.
-        @ip_address        = '52.224.165.15'  # This will change if you had to restart the @device_name.
-        @mismatch_ip       = '52.168.33.118'  # This will change if you had to restart the 'miqmismatch1' VM.
-        @managed_os_disk   = "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd"
-        @managed_data_disk = "miqazure-linux-managed-data-disk"
-        @template          = nil
-        @avail_zone        = nil
+        define_shared_variables
+        @mismatch_ip = '52.168.33.118'
       end
 
       after do

spec/models/manageiq/providers/azure/cloud_manager/deployments_caching_spec.rb

@@ -22,24 +22,7 @@
     }
   ].each do |refresh_settings|
     context "with settings #{refresh_settings}" do
-      before do
-        _guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
-
-        @ems = FactoryGirl.create(:ems_azure_with_vcr_authentication, :zone => zone, :provider_region => 'eastus')
-
-        @resource_group    = 'miq-azure-test1'
-        @managed_vm        = 'miqazure-linux-managed'
-        @device_name       = 'miq-test-rhel1' # Make sure this is running if generating a new cassette.
-        @vm_powered_off    = 'miqazure-centos1' # Make sure this is powered off if generating a new cassette.
-        @ip_address        = '52.224.165.15' # This will change if you had to restart the @device_name.
-        @mismatch_ip       = '13.92.63.10'   # This will change if you had to restart the 'miqmismatch1' VM.
-        @managed_os_disk   = "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd"
-        @managed_data_disk = "miqazure-linux-managed-data-disk"
-        @template          = nil
-        @avail_zone        = nil
-
-        @resource_group_managed_vm = "miq-azure-test4"
-      end
+      before { define_shared_variables }
 
       after do
         ::Azure::Armrest::Configuration.clear_caches

spec/models/manageiq/providers/azure/cloud_manager/refresher_spec.rb

@@ -17,26 +17,7 @@
         )
       end
 
-      before do
-        _guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
-
-        @ems = FactoryGirl.create(:ems_azure_with_vcr_authentication, :zone => zone, :provider_region => 'eastus')
-
-        @resource_group    = 'miq-azure-test1'
-        @managed_vm        = 'miqazure-linux-managed'
-        @device_name       = 'miq-test-rhel1' # Make sure this is running if generating a new cassette.
-        @ip_address        = '52.224.165.15'  # This will change if you had to restart the @device_name.
-        @mismatch_ip       = '13.92.63.10'    # This will change if you had to restart the 'miqmismatch1' VM.
-        @managed_os_disk   = "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd"
-        @managed_data_disk = "miqazure-linux-managed-data-disk"
-        @template          = nil
-        @avail_zone        = nil
-
-        FactoryGirl.create(:tag_mapping_with_category,
-                           :labeled_resource_type => 'VmAzure',
-                           :label_name            => 'Shutdown',
-                           :category_name         => 'azure:vm:shutdown')
-      end
+      before { define_shared_variables }
 
       after do
         ::Azure::Armrest::Configuration.clear_caches
@@ -113,9 +94,6 @@
       end
 
       it "will perform a full refresh" do
-        # NOTE: for backporting compatibility, where tag mapping was not implemented for the Graph refresh
-        is_old_refresh = !refresh_settings[:inventory_object_refresh]
-
         2.times do # Run twice to verify that a second run with existing data does not change anything
           setup_ems_and_cassette(refresh_settings)
 
@@ -126,8 +104,8 @@
           assert_specific_flavor
           assert_specific_disk
           assert_specific_security_group
-          assert_specific_vm_powered_on(is_old_refresh)
-          assert_specific_vm_powered_off(is_old_refresh)
+          assert_specific_vm_powered_on
+          assert_specific_vm_powered_off
           assert_specific_template
           assert_specific_orchestration_template
           assert_specific_orchestration_stack

spec/models/manageiq/providers/azure/cloud_manager/targeted_refresher_scope_spec.rb

@@ -17,24 +17,7 @@
         )
       end
 
-      before do
-        _guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
-
-        @ems = FactoryGirl.create(:ems_azure_with_vcr_authentication, :zone => zone, :provider_region => 'eastus')
-
-        @resource_group    = 'miq-azure-test1'
-        @managed_vm        = 'miqazure-linux-managed'
-        @device_name       = 'miq-test-rhel1' # Make sure this is running if generating a new cassette.
-        @vm_powered_off    = 'miqazure-centos1' # Make sure this is powered off if generating a new cassette.
-        @ip_address        = '52.224.165.15'  # This will change if you had to restart the @device_name.
-        @mismatch_ip       = '13.92.63.10'    # This will change if you had to restart the 'miqmismatch1' VM.
-        @managed_os_disk   = "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd"
-        @managed_data_disk = "miqazure-linux-managed-data-disk"
-        @template          = nil
-        @avail_zone        = nil
-
-        @resource_group_managed_vm = "miq-azure-test4"
-      end
+      before { define_shared_variables }
 
       after do
         ::Azure::Armrest::Configuration.clear_caches

spec/models/manageiq/providers/azure/cloud_manager/targeted_refresher_spec.rb

@@ -23,22 +23,8 @@
           end
 
           before do
-            _guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
-
-            @ems = FactoryGirl.create(:ems_azure_with_vcr_authentication, :zone => zone, :provider_region => 'eastus')
-
-            @resource_group    = 'miq-azure-test1'
-            @managed_vm        = 'miqazure-linux-managed'
-            @device_name       = 'miq-test-rhel1' # Make sure this is running if generating a new cassette.
-            @vm_powered_off    = 'miqazure-centos1' # Make sure this is powered off if generating a new cassette.
-            @ip_address        = '52.224.165.15' # This will change if you had to restart the @device_name.
-            @mismatch_ip       = '52.168.33.118' # This will change if you had to restart the 'miqmismatch1' VM.
-            @managed_os_disk   = "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd"
-            @managed_data_disk = "miqazure-linux-managed-data-disk"
-            @template          = nil
-            @avail_zone        = nil
-
-            @resource_group_managed_vm = "miq-azure-test4"
+            define_shared_variables
+            @mismatch_ip = '52.168.33.118'
           end
 
           after do