Skip to content

Commit

Permalink
Merge pull request #15198 from Ladas/generalize_targeted_inventory_co…
Browse files Browse the repository at this point in the history 
…llection_saving

Generalize targeted inventory collection saving
  • Loading branch information
@agrare
agrare authored May 25, 2017
2 parents b636c8c + 0d0f45d commit 53ddf88
Show file tree
Hide file tree
Showing 4 changed files with 260 additions and 71 deletions.
73 changes: 68 additions & 5 deletions app/models/manager_refresh/inventory_collection.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ class InventoryCollection
:internal_attributes, :delete_method, :data, :data_index, :dependency_attributes, :manager_ref,
:association, :complete, :update_only, :transitive_dependency_attributes, :custom_manager_uuid,
:custom_db_finder, :check_changed, :arel, :builder_params, :loaded_references, :db_data_index,
:inventory_object_attributes, :name, :manager_ref_allowed_nil
:inventory_object_attributes, :name, :parent_inventory_collections, :manager_uuids,
:skeletal_manager_uuids, :targeted_arel, :targeted, :manager_ref_allowed_nil

delegate :each, :size, :to => :to_a

Expand Down Expand Up @@ -285,6 +286,34 @@ class InventoryCollection
# one.
# @param name [Symbol] A unique name of the InventoryCollection under a Persister. If not provided, the :association
# attribute is used. Providing either :name or :association is mandatory.
# @param parent_inventory_collections [Array] Array of symbols having a name of the
# ManagerRefresh::InventoryCollection objects, that serve as parents to this InventoryCollection. Then this
# InventoryCollection completeness will be encapsulated by the parent_inventory_collections :manager_uuids
# instead of this InventoryCollection :manager_uuids.
# @param manager_uuids [Array] Array of manager_uuids of the InventoryObjects we want to create/update/delete. Using
# this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
# case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
# custom :targeted_arel.
# @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
# we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
# manager_uuids available.
# Example:
# targeted_arel = lambda do |inventory_collection|
# # Getting ems_refs of parent :vms and :miq_templates
# manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
# inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
# 'vms' => {:ems_ref => manager_uuids}
# )
# end
#
# inventory_collection = InventoryCollection.new({
# :model_class => ::Hardware,
# :association => :hardwares,
# :parent_inventory_collection => [:vms, :miq_templates],
# :targeted_arel => targeted_arel,
# })
# @param targeted [Boolean] True if the collection is targeted, in that case it will be leveraging :manager_uuids
# :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
# @param manager_ref_allowed_nil [Array] Array of symbols having manager_ref columns, that are a foreign key an can
# be nil. Given the table are shared by many providers, it can happen, that the table is used only partially.
# Then it can happen we want to allow certain foreign keys to be nil, while being sure the referential
Expand All @@ -294,7 +323,9 @@ def initialize(model_class: nil, manager_ref: nil, association: nil, parent: nil
custom_save_block: nil, delete_method: nil, data_index: nil, data: nil, dependency_attributes: nil,
attributes_blacklist: nil, attributes_whitelist: nil, complete: nil, update_only: nil,
check_changed: nil, custom_manager_uuid: nil, custom_db_finder: nil, arel: nil, builder_params: {},
inventory_object_attributes: nil, unique_index_columns: nil, name: nil, manager_ref_allowed_nil: nil)
inventory_object_attributes: nil, unique_index_columns: nil, name: nil,
parent_inventory_collections: nil, manager_uuids: [], targeted_arel: nil, targeted: nil,
manager_ref_allowed_nil: nil)
@model_class = model_class
@manager_ref = manager_ref || [:ems_ref]
@custom_manager_uuid = custom_manager_uuid
Expand All @@ -319,6 +350,13 @@ def initialize(model_class: nil, manager_ref: nil, association: nil, parent: nil

@manager_ref_allowed_nil = manager_ref_allowed_nil || []

# Targeted mode related attributes
@manager_uuids = Set.new.merge(manager_uuids)
@parent_inventory_collections = parent_inventory_collections
@skeletal_manager_uuids = Set.new
@targeted_arel = targeted_arel
@targeted = !!targeted

raise "You have to pass either :name or :association argument to .new of #{self}" if @name.blank?

@inventory_object_attributes = inventory_object_attributes
Expand Down Expand Up @@ -443,6 +481,10 @@ def supports_sti?
@supports_sti_cache
end

def targeted?
targeted
end

def unique_index_columns
return @unique_index_columns if @unique_index_columns

Expand Down Expand Up @@ -694,7 +736,7 @@ def association_to_base_class_mapping
def foreign_keys
return [] unless model_class

@foreign_keys_cache ||= belongs_to_associations.map { |x| x.foreign_key }
@foreign_keys_cache ||= belongs_to_associations.map(&:foreign_key)
end

def fixed_foreign_keys
Expand Down Expand Up @@ -729,20 +771,41 @@ def inspect
to_s
end

def scan!
def scan!(indexed_inventory_collections)
data.each do |inventory_object|
scan_inventory_object!(inventory_object)
end

if targeted? && parent_inventory_collections.present?
self.parent_inventory_collections = parent_inventory_collections.map do |inventory_collection_index|
inventory_collection = indexed_inventory_collections[inventory_collection_index]
raise "Cannot find inventory collection #{inventory_collection_index} from #{self}" unless inventory_collection
inventory_collection
end
end
end

def db_collection_for_comparison
if targeted?
if targeted_arel.respond_to?(:call)
targeted_arel.call(self)
else
raise "Can't build :targeted_arel for #{self}, please provide it as an argument." if manager_ref.count > 1
full_collection_for_comparison.where(manager_ref.first => (manager_uuids + skeletal_manager_uuids).to_a.flatten.compact)
end
else
full_collection_for_comparison
end
end

def full_collection_for_comparison
return arel unless arel.nil?
parent.send(association)
end

private

attr_writer :attributes_blacklist, :attributes_whitelist, :db_data_index
attr_writer :attributes_blacklist, :attributes_whitelist, :db_data_index, :parent_inventory_collections

# Finds manager_uuid in the DB. Using a configured strategy we cache obtained data in the db_data_index, so the
# same find will not hit database twice. Also if we use lazy_links and this is called when
Expand Down
4 changes: 3 additions & 1 deletion app/models/manager_refresh/inventory_collection/scanner.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ class << self
#
# @param inventory_collections [Array] Array fo
def scan!(inventory_collections)
indexed_inventory_collections = inventory_collections.index_by(&:name)

inventory_collections.each do |inventory_collection|
inventory_collection.data_collection_finalized = true
inventory_collection.scan!
inventory_collection.scan!(indexed_inventory_collections)
end
end
end
Expand Down
130 changes: 92 additions & 38 deletions app/models/manager_refresh/inventory_collection_default/cloud_manager.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,10 @@ def key_pairs(extra_attributes = {})

def hardwares(extra_attributes = {})
attributes = {
:model_class => ::Hardware,
:manager_ref => [:vm_or_template],
:association => :hardwares
:model_class => ::Hardware,
:manager_ref => [:vm_or_template],
:association => :hardwares,
:parent_inventory_collections => [:vms, :miq_templates],
}

attributes[:custom_manager_uuid] = lambda do |hardware|
Expand All @@ -92,14 +93,22 @@ def hardwares(extra_attributes = {})
relation
end

attributes[:targeted_arel] = lambda do |inventory_collection|
manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a }
inventory_collection.parent.hardwares.joins(:vm_or_template).where(
'vms' => {:ems_ref => manager_uuids}
)
end

attributes.merge!(extra_attributes)
end

def disks(extra_attributes = {})
attributes = {
:model_class => ::Disk,
:manager_ref => [:hardware, :device_name],
:association => :disks
:model_class => ::Disk,
:manager_ref => [:hardware, :device_name],
:association => :disks,
:parent_inventory_collections => [:vms],
}

if extra_attributes[:strategy] == :local_db_cache_all
Expand All @@ -108,14 +117,22 @@ def disks(extra_attributes = {})
end
end

attributes[:targeted_arel] = lambda do |inventory_collection|
manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a }
inventory_collection.parent.disks.joins(:hardware => :vm_or_template).where(
:hardware => {'vms' => {:ems_ref => manager_uuids}}
)
end

attributes.merge!(extra_attributes)
end

def networks(extra_attributes = {})
attributes = {
:model_class => ::Network,
:manager_ref => [:hardware, :description],
:association => :networks
:model_class => ::Network,
:manager_ref => [:hardware, :description],
:association => :networks,
:parent_inventory_collections => [:vms],
}

if extra_attributes[:strategy] == :local_db_cache_all
Expand All @@ -124,6 +141,13 @@ def networks(extra_attributes = {})
end
end

extra_attributes[:targeted_arel] = lambda do |inventory_collection|
manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a }
inventory_collection.parent.networks.joins(:hardware => :vm_or_template).where(
:hardware => {'vms' => {:ems_ref => manager_uuids}}
)
end

attributes.merge!(extra_attributes)
end

Expand All @@ -142,28 +166,52 @@ def orchestration_stacks(extra_attributes = {})

def orchestration_stacks_resources(extra_attributes = {})
attributes = {
:model_class => ::OrchestrationStackResource,
:association => :orchestration_stacks_resources,
:model_class => ::OrchestrationStackResource,
:association => :orchestration_stacks_resources,
:parent_inventory_collections => [:orchestration_stacks]
}

extra_attributes[:targeted_arel] = lambda do |inventory_collection|
manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a }
inventory_collection.parent.orchestration_stacks_resources.references(:orchestration_stacks).where(
:orchestration_stacks => {:ems_ref => manager_uuids}
)
end

attributes.merge!(extra_attributes)
end

def orchestration_stacks_outputs(extra_attributes = {})
attributes = {
:model_class => ::OrchestrationStackOutput,
:association => :orchestration_stacks_outputs
:model_class => ::OrchestrationStackOutput,
:association => :orchestration_stacks_outputs,
:parent_inventory_collections => [:orchestration_stacks],
}

extra_attributes[:targeted_arel] = lambda do |inventory_collection|
manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a }
inventory_collection.parent.orchestration_stacks_outputs.references(:orchestration_stacks).where(
:orchestration_stacks => {:ems_ref => manager_uuids}
)
end

attributes.merge!(extra_attributes)
end

def orchestration_stacks_parameters(extra_attributes = {})
attributes = {
:model_class => ::OrchestrationStackParameter,
:association => :orchestration_stacks_parameters
:model_class => ::OrchestrationStackParameter,
:association => :orchestration_stacks_parameters,
:parent_inventory_collections => [:orchestration_stacks],
}

extra_attributes[:targeted_arel] = lambda do |inventory_collection|
manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a }
inventory_collection.parent.orchestration_stacks_parameters.references(:orchestration_stacks).where(
:orchestration_stacks => {:ems_ref => manager_uuids}
)
end

attributes.merge!(extra_attributes)
end

Expand Down Expand Up @@ -204,11 +252,12 @@ def orchestration_stack_ancestry(extra_attributes = {})
.select([:id, :ancestry])
.where(:id => stacks_parents.values).find_each.index_by(&:id)

model_class
.select([:id, :ancestry])
.where(:id => stacks_parents.keys).find_each do |stack|
parent = stacks_parents_indexed[stacks_parents[stack.id]]
stack.update_attribute(:parent, parent)
ActiveRecord::Base.transaction do
model_class.select([:id, :ancestry])
.where(:id => stacks_parents.keys).find_each do |stack|
parent = stacks_parents_indexed[stacks_parents[stack.id]]
stack.update_attribute(:parent, parent)
end
end
end

Expand Down Expand Up @@ -241,25 +290,30 @@ def vm_and_miq_template_ancestry(extra_attributes = {})
end
end

# associate parent templates to child instances
parent_miq_templates = miq_templates_inventory_collection.model_class
.select([:id])
.where(:id => vms_genealogy_parents.values).find_each.index_by(&:id)
vms_inventory_collection.model_class
.select([:id])
.where(:id => vms_genealogy_parents.keys).find_each do |vm|
parent = parent_miq_templates[vms_genealogy_parents[vm.id]]
vm.with_relationship_type('genealogy') { vm.parent = parent }
ActiveRecord::Base.transaction do
# associate parent templates to child instances
parent_miq_templates = miq_templates_inventory_collection.model_class
.select([:id])
.where(:id => vms_genealogy_parents.values).find_each.index_by(&:id)
vms_inventory_collection.model_class
.select([:id])
.where(:id => vms_genealogy_parents.keys).find_each do |vm|
parent = parent_miq_templates[vms_genealogy_parents[vm.id]]
vm.with_relationship_type('genealogy') { vm.parent = parent }
end
end
# associate parent instances to child templates
parent_vms = vms_inventory_collection.model_class
.select([:id])
.where(:id => miq_template_genealogy_parents.values).find_each.index_by(&:id)
miq_templates_inventory_collection.model_class
.select([:id])
.where(:id => miq_template_genealogy_parents.keys).find_each do |miq_template|
parent = parent_vms[miq_template_genealogy_parents[miq_template.id]]
miq_template.with_relationship_type('genealogy') { miq_template.parent = parent }

ActiveRecord::Base.transaction do
# associate parent instances to child templates
parent_vms = vms_inventory_collection.model_class
.select([:id])
.where(:id => miq_template_genealogy_parents.values).find_each.index_by(&:id)
miq_templates_inventory_collection.model_class
.select([:id])
.where(:id => miq_template_genealogy_parents.keys).find_each do |miq_template|
parent = parent_vms[miq_template_genealogy_parents[miq_template.id]]
miq_template.with_relationship_type('genealogy') { miq_template.parent = parent }
end
end
end

Expand Down
Loading

0 comments on commit 53ddf88

Please sign in to comment.