From 5688e61a6d5bf6a730ec7a62d018f89295f7a598 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Tue, 27 Jun 2023 09:52:49 -0400 Subject: [PATCH] VimPerformanceState#capture - don't capture old deleted records VimPerformanceState records only capture active objects and those deleted after this hour's time period This cuts down on the number of records processed. (by a lot) Numbers ======= ``` VimPerformanceState.first ; ExtManagementSystem.first ; ContainerImage.first ; Container.first ; nil bookend(gc:true) { Metric::Rollup.rollup_child_metrics(ContainerImage.find(627), Time.now.utc.iso8601, "historical", :containers) } ``` Numbers run for rollup_child_metrics for a parent object that hasn't been created yet. If the parent object has been run, then we don't need a capture, and we'll see no difference | ms | bytes | objects |query | qry ms | rows |` comments` | ---:| ---:| ---:| ---:| ---:| ---:| --- | 2,131.6 | 77,558,748* | 1,070,108 | 13 | 1,760.5 | 13,680 |`rollup-create-vps-before` | 1,632.5 | 5,139,458* | 393,268 | 13 | 1,517.7 | 174 |`rollup-create-vps-after` * Memory usage does not reflect 175,949 freed objects. * Memory usage does not reflect 59,056 freed objects. We've already merged code to reduce the number of records returned from VimPerformanceState#containers So removing archived from the VimPerformanceState will reduce the size of the query sent to fetch containers. This will reduce the query ms (simpler/smaller query), but this does not reduce the number of records returned The savings there is due to the fact that capture does not download all the archived records. So this commit saves all the id downloads, and the commit from the fetch pr (just merged a few commits back), reduces all archived records fetch as part of the capture process --- app/models/vim_performance_state.rb | 4 +++- spec/models/vim_performance_state_spec.rb | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/models/vim_performance_state.rb b/app/models/vim_performance_state.rb index 0691724c7c9..783853f5067 100644 --- a/app/models/vim_performance_state.rb +++ b/app/models/vim_performance_state.rb @@ -228,7 +228,9 @@ def fetch_assoc_records(resource, assoc) end return false unless resource.respond_to?(method) - resource.send(method) + records = resource.send(method) + records = records.not_archived_before(timestamp) if records.try(:klass).respond_to?(:not_archived_before) + records end def capture_parent_cluster diff --git a/spec/models/vim_performance_state_spec.rb b/spec/models/vim_performance_state_spec.rb index d5448db5281..9ee9b8e71b5 100644 --- a/spec/models/vim_performance_state_spec.rb +++ b/spec/models/vim_performance_state_spec.rb @@ -11,7 +11,18 @@ end end - describe ".capture_host_sockets" do + describe "#capture_assoc_ids" do + it "captures running containers" do + container_image = FactoryBot.create(:container_image) + containers = create_past_present_future(:container, {:container_image => container_image}) + + state = VimPerformanceState.capture(container_image) + expect(state.get_assoc(:containers).map(&:to_i)).to match_array(containers.map(&:id)) + expect(state.containers.map(&:id)).to match_array(containers.map(&:id)) + end + end + + describe "#capture_host_sockets" do it "returns the host sockets when given a host" do hardware = FactoryBot.build(:hardware, :cpu_sockets => 2) host = FactoryBot.create(:host, :hardware => hardware) @@ -151,6 +162,15 @@ expect(state.timestamp).to be_within(1.minute).of(capture_time) end + + it "fetches only containers deleted after capture" do + container_image = FactoryBot.create(:container_image) + containers = create_past_present_future(:container, {:container_image => container_image}) + + results = fetch_records(container_image, :containers, capture_time) + expect(results).to eq(containers) + state = VimPerformanceState.capture(container_image) + end end private