Merge pull request #158 from djberg96/disk_information

Correct and update disk information

app/models/manageiq/providers/azure/cloud_manager/refresh_parser.rb

@@ -23,6 +23,7 @@ def initialize(ems, options = Config::Options.new)
         @tds               = template_deployment_service(@config)
         @rgs               = resource_group_service(@config)
         @sas               = storage_account_service(@config)
+        @sds               = storage_disk_service(@config)
         @mis               = managed_image_service(@config)
         @vmis              = virtual_machine_image_service(@config, :location => @ems.provider_region)
         @options           = options || {}
@@ -40,6 +41,8 @@ def ems_inv_to_hashes
         _log.info("#{log_header}...")
         get_resource_groups
         get_series
+        get_managed_disks
+        get_unmanaged_storage
         get_availability_zones
         get_stacks
         get_stack_templates
@@ -54,6 +57,14 @@ def ems_inv_to_hashes
 
       private
 
+      def get_managed_disks
+        @managed_disks ||= @sds.list_all
+      end
+
+      def get_unmanaged_storage
+        @storage_accounts ||= @sas.list_all
+      end
+
       def get_resource_groups
         groups = collect_inventory(:resource_groups) { resource_groups }
         process_collection(groups, :resource_groups) do |resource_group|
@@ -305,18 +316,10 @@ def guest_os(instance)
       def populate_hardware_hash_with_disks(hardware_disks_array, instance)
         data_disks = instance.properties.storage_profile.data_disks
         data_disks.each do |disk|
-          disk_size      = disk.respond_to?(:disk_size_gb) ? disk.disk_size_gb * 1.gigabyte : 0
-          disk_name      = disk.name
-          disk_location  = disk.try(:vhd).try(:uri) || disk.try(:managed_disk).try(:id)
-
-          add_instance_disk(hardware_disks_array, disk_size, disk_name, disk_location)
+          add_instance_disk(hardware_disks_array, instance, disk)
         end
       end
 
-      def add_instance_disk(disks, size, name, location)
-        super(disks, size, location, name, "azure")
-      end
-
       # TODO(lsmola) NetworkManager, storing IP addresses under hardware/network will go away, once all providers are
       # unified under the NetworkManager
       def get_hardware_network_info(instance)
@@ -350,13 +353,52 @@ def populate_hardware_hash_with_series_attributes(hardware_hash, instance, serie
         hardware_hash[:memory_mb]       = series[:memory] / 1.megabyte
         hardware_hash[:disk_capacity]   = series[:root_disk_size] + series[:swap_disk_size]
 
-        os_disk = instance.properties.storage_profile.os_disk
-        sz      = series[:root_disk_size]
-        vhd_loc = os_disk.try(:vhd).try(:uri) || os_disk.try(:managed_disk).try(:id)
+        disk = instance.properties.storage_profile.os_disk
+        add_instance_disk(hardware_hash[:disks], instance, disk)
+      end
 
-        add_instance_disk(hardware_hash[:disks], sz, os_disk.name, vhd_loc) unless sz.zero?
+      # Redefine the inherited method for our purposes
+      def add_instance_disk(array, instance, disk)
+        if instance.managed_disk?
+          disk_type     = 'managed'
+          disk_location = disk.managed_disk.id
+          managed_disk  = @managed_disks.find { |d| d.id.casecmp(disk_location).zero? }
+          disk_size     = managed_disk.properties.disk_size_gb.gigabytes
+          mode          = managed_disk.sku.name
+        else
+          disk_type     = 'unmanaged'
+          disk_location = disk.try(:vhd).try(:uri)
+          disk_size     = disk.try(:disk_size_gb).try(:gigabytes)
+
+          if disk_location
+            uri = Addressable::URI.parse(disk_location)
+            storage_name = uri.host.split('.').first
+            container_name = File.dirname(uri.path)
+            blob_name = uri.basename
+
+            storage_acct = @storage_accounts.find { |s| s.name.casecmp(storage_name).zero? }
+            mode = storage_acct.sku.name
+
+            if @options.get_unmanaged_disk_space && disk_size.nil?
+              storage_keys = @sas.list_account_keys(storage_acct.name, storage_acct.resource_group)
+              storage_key  = storage_keys['key1'] || storage_keys['key2']
+              blob_props   = storage_acct.blob_properties(container_name, blob_name, storage_key)
+              disk_size    = blob_props.content_length.to_i
+            end
+          end
+        end
+
+        disk_record = {
+          :device_type     => 'disk',
+          :controller_type => 'azure',
+          :device_name     => disk.name,
+          :location        => disk_location,
+          :size            => disk_size,
+          :disk_type       => disk_type,
+          :mode            => mode
+        }
 
-        # No data availbale on swap disk? Called temp or resource disk.
+        array << disk_record
       end
 
       def parse_stack(deployment)

app/models/manageiq/providers/azure/refresh_helper_methods.rb

@@ -144,6 +144,12 @@ def template_deployment_service(config)
     end
   end
 
+  def storage_disk_service(config)
+    ::Azure::Armrest::Storage::DiskService.new(config).tap do |service|
+      service.api_version = Settings.ems.ems_azure.api_versions.storage_disk.to_s
+    end
+  end
+
   def storage_account_service(config)
     ::Azure::Armrest::StorageAccountService.new(config).tap do |service|
       service.api_version = Settings.ems.ems_azure.api_versions.storage_account.to_s

config/settings.yml

@@ -8,8 +8,10 @@
       :managed_image: 2017-03-30
       :network_interface: 2017-03-01
       :network_security_group: 2016-12-01
+      :resource: 2016-09-01
       :resource_group: 2016-09-01
       :storage_account: 2016-12-01
+      :storage_disk: 2017-03-30
       :template_deployment: 2016-09-01
       :virtual_machine: 2017-03-30
       :virtual_network: 2017-03-01
@@ -38,6 +40,8 @@
       - MicrosoftWindowsServer:WindowsServer-HUB:2016-Datacenter-HUB:2016.127.20170630
       - OpenLogic:CentOS:7.3:7.3.20170517
       - RedHat:RHEL:7.3:7.3.2017051117
+    # Collecting disk information on unmanaged VM's slows down the refresh.
+    :get_unmanaged_disk_space: true
 
 :http_proxy:
   :azure:

spec/models/manageiq/providers/azure/cloud_manager/refresher_spec.rb

@@ -9,8 +9,8 @@
     @resource_group    = 'miq-azure-test1'
     @managed_vm        = 'miqazure-linux-managed'
     @device_name       = 'miq-test-rhel1' # Make sure this is running if generating a new cassette.
-    @ip_address        = '40.76.6.142' # This will change if you had to restart the @device_name.
-    @mismatch_ip       = '40.76.203.26' # This will change if you had to restart the 'miqmismatch' VM.
+    @ip_address        = '40.76.35.39' # This will change if you had to restart the @device_name.
+    @mismatch_ip       = '40.76.46.32' # This will change if you had to restart the 'miqmismatch' VM.
     @managed_os_disk   = "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd"
     @managed_data_disk = "miqazure-linux-managed-data-disk"
     @template          = nil
@@ -134,7 +134,7 @@ def setup_ems_and_cassette
   def expected_table_counts
     {
       :ext_management_system         => 2,
-      :flavor                        => 133,
+      :flavor                        => 144,
       :availability_zone             => 1,
       :vm_or_template                => 14,
       :vm                            => 13,
@@ -495,7 +495,7 @@ def assert_specific_disk
 
     expect(disk).to have_attributes(
       :location => "https://miqazuretest18686.blob.core.windows.net/vhds/miq-test-rhel12016218112243.vhd",
-      :size     => 1023.megabyte
+      :size     => 32212255232 # 30gb, approx
     )
   end
 
@@ -510,7 +510,7 @@ def assert_specific_managed_disk
     expect(disk.location).to eql("/subscriptions/#{@ems.subscription}/resourceGroups/"\
                                   "MIQ-AZURE-TEST4/providers/Microsoft.Compute/disks/"\
                                   "miqazure-linux-managed_OsDisk_1_7b2bdf790a7d4379ace2846d307730cd")
-    expect(disk.size).to eql(1023.megabyte)
+    expect(disk.size).to eql(32.gigabytes)
   end
 
   def assert_specific_resource_group