Skip to content

Commit

Permalink
Collect node custom attributes from hawkular during refresh
Browse files Browse the repository at this point in the history 
Use Hawkular client to fetch metrics with miq_metric tag and save each metric as additional attribute in the relevant node
Use store_path and fetch_path to parse additional attributes
Add hawk collected custom attribute test in refresher_spec.rb
Change metrics_capture/capture_context_spec.rb and related vcr cassetes to work with new refresh.
  • Loading branch information
@zgalor
zgalor committed Jan 25, 2017
1 parent e64632e commit dbc58da
Show file tree
Hide file tree
Showing 15 changed files with 17,137 additions and 11,469 deletions.
1 change: 1 addition & 0 deletions app/models/container_node.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class ContainerNode < ApplicationRecord
has_many :container_routes, -> { distinct }, :through => :container_services
has_many :container_replicators, -> { distinct }, :through => :container_groups
has_many :labels, -> { where(:section => "labels") }, :class_name => "CustomAttribute", :as => :resource, :dependent => :destroy
has_many :additional_attributes, -> { where(:section => "additional_attributes") }, :class_name => "CustomAttribute", :as => :resource, :dependent => :destroy
has_one :computer_system, :as => :managed_entity, :dependent => :destroy
belongs_to :lives_on, :polymorphic => true
has_one :hardware, :through => :computer_system
Expand Down
16 changes: 15 additions & 1 deletion app/models/ems_refresh/save_inventory_container.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,8 @@ def save_container_nodes_inventory(ems, hashes, target = nil)
end

save_inventory_multi(ems.container_nodes, hashes, deletes, [:ems_ref],
[:labels, :tags, :computer_system, :container_conditions], [:namespace])
[:labels, :tags, :computer_system, :container_conditions,
:additional_attributes], [:namespace])
store_ids_for_new_records(ems.container_nodes, hashes, :ems_ref)
end

Expand Down Expand Up @@ -395,6 +396,19 @@ def save_container_volumes_inventory(container_group, hashes, target = nil)
store_ids_for_new_records(container_group.container_volumes, hashes, :name)
end

def save_additional_attributes_inventory(entity, hashes, target = nil)
return if hashes.nil?

deletes = if target.kind_of?(ExtManagementSystem)
:use_association
else
[]
end

save_inventory_multi(entity.additional_attributes, hashes, deletes, [:section, :name])
store_ids_for_new_records(entity.additional_attributes, hashes, [:section, :name])
end

def save_custom_attribute_attribute_inventory(entity, attribute_name, hashes, target = nil)
return if hashes.nil?

Expand Down
1 change: 1 addition & 0 deletions ...models/manageiq/providers/kubernetes/container_manager/metrics_capture/hawkular_client.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ def initialize(ext_management_system, tenant = '_system')
end

delegate :gauges, :to => :hawkular_client
delegate :strings, :to => :hawkular_client
end
28 changes: 28 additions & 0 deletions app/models/manageiq/providers/kubernetes/container_manager/refresh_parser.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ def initialize
end

def ems_inv_to_hashes(inventory)
get_additional_attributes(inventory)
get_nodes(inventory)
get_namespaces(inventory)
get_resource_quotas(inventory)
Expand Down Expand Up @@ -125,6 +126,19 @@ def get_component_statuses(inventory)
end
end

def get_additional_attributes(inventory)
inventory["additional_attributes"] ||= {}
process_collection(inventory["additional_attributes"], :additional_attributes) do |aa|
parse_additional_attribute(aa)
end

@data[:additional_attributes].each do |aa|
ats = @data_index.fetch_path(:additional_attributes, :by_node, aa[:node]) || []
ats << {:name => aa[:name], :value => aa[:value], :section => "additional_attributes"}
@data_index.store_path(:additional_attributes, :by_node, aa[:node], ats)
end
end

def process_collection(collection, key, &block)
@data[key] ||= []
collection.each { |item| process_collection_item(item, key, &block) }
Expand Down Expand Up @@ -189,6 +203,18 @@ def cross_link_node(new_result)
new_result[:lives_on_type] = host_instance.try(:type)
end

def parse_additional_attribute(attribute)
# Assuming keys are in format "node/<hostname.example.com/key"
if attribute[0] && attribute[0].split("/").count == 3
{ attribute[0].split("/").first.to_sym => attribute[0].split("/").second,
:name => attribute[0].split("/").last,
:value => attribute[1],
:section => "additional_attributes"}
else
{}
end
end

def parse_node(node)
new_result = parse_base_item(node)

Expand Down Expand Up @@ -234,6 +260,8 @@ def parse_node(node)
new_result[:container_conditions] = parse_conditions(node)
cross_link_node(new_result)

new_result[:additional_attributes] = @data_index.fetch_path(:additional_attributes, :by_node, node.metadata.name)

new_result
end

Expand Down
13 changes: 13 additions & 0 deletions app/models/manageiq/providers/openshift/container_manager/refresher.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,18 @@ class ContainerManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
{:name => 'images'}
]

def fetch_hawk_inv(ems)
hawk = ManageIQ::Providers::Kubernetes::ContainerManager::MetricsCapture::HawkularClient.new(ems, '_ops')
keys = hawk.strings.query(:miq_metric => true)
keys.each_with_object({}) do |k, attributes|
values = hawk.strings.get_data(k.json["id"], :limit => 1, :order => "DESC")
attributes[k.json["id"]] = values.first["value"] unless values.empty?
end
rescue => err
_log.error err.message
return nil
end

def parse_legacy_inventory(ems)
kube_entities = ems.with_provider_connection(:service => KUBERNETES_EMS_TYPE) do |kubeclient|
fetch_entities(kubeclient, KUBERNETES_ENTITIES)
Expand All @@ -20,6 +32,7 @@ def parse_legacy_inventory(ems)
fetch_entities(openshift_client, OPENSHIFT_ENTITIES)
end
entities = openshift_entities.merge(kube_entities)
entities["additional_attributes"] = fetch_hawk_inv(ems) || {}
EmsRefresh.log_inv_debug_trace(entities, "inv_hash:")
ManageIQ::Providers::Openshift::ContainerManager::RefreshParser.ems_inv_to_hashes(entities)
end
Expand Down
5 changes: 3 additions & 2 deletions ...s/manageiq/providers/kubernetes/container_manager/metrics_capture/capture_context_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@
allow(MiqServer).to receive(:my_zone).and_return("default")
hostname = 'capture.context.com'
token = 'theToken'
@start_time = 1_468_235_843
@end_time = 1_468_235_843

@start_time = 1_482_306_073
@end_time = 1_482_306_073
@interval = 20

@ems = FactoryGirl.create(
Expand Down
202 changes: 197 additions & 5 deletions spec/models/manageiq/providers/kubernetes/container_manager/refresh_parser_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,8 @@
},
:namespace => nil,
:resource_version => '369104',
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode'
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode',
:additional_attributes => nil
})
end

Expand Down Expand Up @@ -823,7 +824,8 @@
},
:namespace => nil,
:resource_version => '3691041',
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode'
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode',
:additional_attributes => nil
})
end

Expand Down Expand Up @@ -866,11 +868,201 @@
:kernel_version => nil
}
},
:namespace => nil,
:resource_version => '369104',
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode'
:namespace => nil,
:resource_version => '369104',
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode',
:additional_attributes => nil
})
end
it "handles node with single custom attribute" do
parser.get_additional_attributes(
"additional_attributes" => { "node/test-node/key" => "val" }
)

expect(
parser.send(
:parse_node,
RecursiveOpenStruct.new(
:metadata => {
:name => 'test-node',
:uid => 'f0c1fe7e-9c09-11e5-bb22-28d2447dcefe',
:resourceVersion => '369104',
:creationTimestamp => '2016-01-01T11:10:21Z'
},
:spec => {
:providerID => 'aws:///zone/aws-id'
},
:status => {
:capacity => {}
}
),
)
).to eq(
:name => 'test-node',
:ems_ref => 'f0c1fe7e-9c09-11e5-bb22-28d2447dcefe',
:ems_created_on => '2016-01-01T11:10:21Z',
:container_conditions => [],
:identity_infra => 'aws:///zone/aws-id',
:labels => [],
:tags => [],
:lives_on_id => nil,
:lives_on_type => nil,
:max_container_groups => nil,
:computer_system => {
:hardware => {
:cpu_total_cores => nil,
:memory_mb => nil
},
:operating_system => {
:distribution => nil,
:kernel_version => nil
}
},
:namespace => nil,
:resource_version => '369104',
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode',
:additional_attributes => [{ :name => "key", :value => "val", :section => "additional_attributes" }]
)
end
it "handles node with multiple custom attributes" do
parser.get_additional_attributes(
"additional_attributes" => { "node/test-node/key1" => "val1",
"node/test-node/key2" => "val2"}
)

expect(
parser.send(
:parse_node,
RecursiveOpenStruct.new(
:metadata => {
:name => 'test-node',
:uid => 'f0c1fe7e-9c09-11e5-bb22-28d2447dcefe',
:resourceVersion => '369104',
:creationTimestamp => '2016-01-01T11:10:21Z'
},
:spec => {
:providerID => 'aws:///zone/aws-id'
},
:status => {
:capacity => {}
}
),
)
).to eq(
:name => 'test-node',
:ems_ref => 'f0c1fe7e-9c09-11e5-bb22-28d2447dcefe',
:ems_created_on => '2016-01-01T11:10:21Z',
:container_conditions => [],
:identity_infra => 'aws:///zone/aws-id',
:labels => [],
:tags => [],
:lives_on_id => nil,
:lives_on_type => nil,
:max_container_groups => nil,
:computer_system => {
:hardware => {
:cpu_total_cores => nil,
:memory_mb => nil
},
:operating_system => {
:distribution => nil,
:kernel_version => nil
}
},
:namespace => nil,
:resource_version => '369104',
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode',
:additional_attributes => [{ :name => "key1", :value => "val1", :section => "additional_attributes" },
{ :name => "key2", :value => "val2", :section => "additional_attributes" }]
)
end
it "ignores custom attributes of a different node" do
parser.get_additional_attributes(
"additional_attributes" => { "node/test-node1/key1" => "val1",
"node/test-node2/key2" => "val2"}
)

expect(
parser.send(
:parse_node,
RecursiveOpenStruct.new(
:metadata => {
:name => 'test-node1',
:uid => 'f0c1fe7e-9c09-11e5-bb22-28d2447dcefe',
:resourceVersion => '369104',
:creationTimestamp => '2016-01-01T11:10:21Z'
},
:spec => {
:providerID => 'aws:///zone/aws-id'
},
:status => {
:capacity => {}
}
),
)
).to eq(
:name => 'test-node1',
:ems_ref => 'f0c1fe7e-9c09-11e5-bb22-28d2447dcefe',
:ems_created_on => '2016-01-01T11:10:21Z',
:container_conditions => [],
:identity_infra => 'aws:///zone/aws-id',
:labels => [],
:tags => [],
:lives_on_id => nil,
:lives_on_type => nil,
:max_container_groups => nil,
:computer_system => {
:hardware => {
:cpu_total_cores => nil,
:memory_mb => nil
},
:operating_system => {
:distribution => nil,
:kernel_version => nil
}
},
:namespace => nil,
:resource_version => '369104',
:type => 'ManageIQ::Providers::Kubernetes::ContainerManager::ContainerNode',
:additional_attributes => [{ :name => "key1", :value => "val1", :section => "additional_attributes" }]
)
end
end

describe "parse_additional_attribute" do
it "parses node attribute" do
expect(
parser.send(
:parse_additional_attribute,
%w(node/test-node/key val)
)
).to eq(:node => "test-node", :name => "key", :value => "val", :section => "additional_attributes")
end
it "parses pod attribute" do
expect(
parser.send(
:parse_additional_attribute,
%w(pod/test-pod/key val)
)
).to eq(:pod => "test-pod", :name => "key", :value => "val", :section => "additional_attributes")
end
it "parses empty attribute" do
expect(
parser.send(
:parse_additional_attribute,
[]
)
).to eq({})
end

it "parses wrong format" do
expect(
parser.send(
:parse_additional_attribute,
%w(key1 val1)
)
).to eq({})
end
end

describe "parse_persistent_volume" do
Expand Down
Loading

0 comments on commit dbc58da

Please sign in to comment.