From 1773711529be641b2dc080921c57847b4150e0d5 Mon Sep 17 00:00:00 2001 From: Joe Rafaniello Date: Tue, 17 Jan 2017 16:28:32 -0500 Subject: [PATCH 1/3] Add embedded ansible role https://www.pivotaltracker.com/story/show/135450841 --- db/fixtures/server_roles.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/db/fixtures/server_roles.csv b/db/fixtures/server_roles.csv index 7ce2ff9b6d0..752cb8b1279 100644 --- a/db/fixtures/server_roles.csv +++ b/db/fixtures/server_roles.csv @@ -2,6 +2,7 @@ name,description,max_concurrent,external_failover,role_scope automate,Automation Engine,0,false,region database_operations,Database Operations,0,false,region database_owner,Database Owner,1,false,database +embedded_ansible,Embedded Ansible,1,false,region ems_inventory,Provider Inventory,1,false,zone ems_metrics_collector,Capacity & Utilization Data Collector,0,false,zone ems_metrics_coordinator,Capacity & Utilization Coordinator,1,false,zone From d7a9ff39a78d82043b3f297b8aa8ff855c7cc0af Mon Sep 17 00:00:00 2001 From: Joe Rafaniello Date: Tue, 17 Jan 2017 16:30:59 -0500 Subject: [PATCH 2/3] Extract forking/process creation to a method subclasses can override. https://www.pivotaltracker.com/story/show/135450841 --- app/models/miq_worker.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/miq_worker.rb b/app/models/miq_worker.rb index 45ad3ca1506..5cb7270c109 100644 --- a/app/models/miq_worker.rb +++ b/app/models/miq_worker.rb @@ -336,7 +336,7 @@ def self.close_pg_sockets_inherited_from_parent end end - def start + def start_runner self.class.before_fork pid = fork(:cow_friendly => true) do self.class.after_fork @@ -345,7 +345,11 @@ def start end Process.detach(pid) - self.pid = pid + pid + end + + def start + self.pid = start_runner save msg = "Worker started: ID [#{id}], PID [#{pid}], GUID [#{guid}]" From 23af6600724fa8d0388bbdca7c377454cc1b3507 Mon Sep 17 00:00:00 2001 From: Joe Rafaniello Date: Tue, 17 Jan 2017 16:37:43 -0500 Subject: [PATCH 3/3] Add embedded ansible worker. Right now, this is a worker that doesn't have a separate process. Instead, it's just a thread running in the server that monitors the supervisord process and handles restarting it when it goes down. https://www.pivotaltracker.com/story/show/135450841 --- app/models/embedded_ansible_worker.rb | 22 ++++++ app/models/embedded_ansible_worker/runner.rb | 77 +++++++++++++++++++ .../worker_management/monitor/class_names.rb | 2 + app/models/miq_worker.rb | 4 + config/settings.yml | 3 + spec/factories/miq_worker.rb | 4 + .../embedded_ansible_worker/runner_spec.rb | 40 ++++++++++ 7 files changed, 152 insertions(+) create mode 100644 app/models/embedded_ansible_worker.rb create mode 100644 app/models/embedded_ansible_worker/runner.rb create mode 100644 spec/models/embedded_ansible_worker/runner_spec.rb diff --git a/app/models/embedded_ansible_worker.rb b/app/models/embedded_ansible_worker.rb new file mode 100644 index 00000000000..6dda36ae733 --- /dev/null +++ b/app/models/embedded_ansible_worker.rb @@ -0,0 +1,22 @@ +class EmbeddedAnsibleWorker < MiqWorker + require_nested :Runner + + self.required_roles = ['embedded_ansible'] + + def start_runner + self.class::Runner.start_worker(worker_options) + # TODO: return supervisord pid + end + + def kill + # Does the base class's kill -9 work on the supervisord process as we want? + end + + def status_update + # don't monitor the memory/cpu usage of this process yet + # If we don't have a pid of a process we want to monitor,super will catch an Errno::ESRCH and abort the worker + end + + # Base class methods we override since we don't have a separate process. We might want to make these opt-in features in the base class that this subclass can choose to opt-out. + def release_db_connection; end +end diff --git a/app/models/embedded_ansible_worker/runner.rb b/app/models/embedded_ansible_worker/runner.rb new file mode 100644 index 00000000000..6e8a0d628dd --- /dev/null +++ b/app/models/embedded_ansible_worker/runner.rb @@ -0,0 +1,77 @@ +class EmbeddedAnsibleWorker::Runner < MiqWorker::Runner + def prepare + update_embedded_ansible_manager + + Thread.new do + setup_ansible + started_worker_record + end + + self + end + + # This thread runs forever until a stop request is received, which with send us to do_exit to exit our thread + def do_work_loop + Thread.new do + _log.info("waiting for ansible to start...") + loop do + # handle if the ansible setup blew up or timed out + break if worker.reload.started? + heartbeat + send(poll_method) + end + + _log.info("entering ansible monitor loop") + loop do + heartbeat + do_work + send(poll_method) + end + end + end + + def setup_ansible + _log.info("calling EmbeddedAnsible.configure") + EmbeddedAnsible.configure unless EmbeddedAnsible.configured? + + _log.info("calling EmbeddedAnsible.start") + EmbeddedAnsible.start + _log.info("calling EmbeddedAnsible.start finished") + end + + def do_work + if EmbeddedAnsible.running? + _log.info("#{log_prefix} supervisord is ok!") + else + _log.warn("#{log_prefix} supervisord is not running, restarting!") + EmbeddedAnsible.start + end + end + + # Because we're running in a thread on the Server + # we need to intercept SystemExit and exit our thread, + # not the main server thread! + def do_exit(*args) + # ensure this doesn't fail or that we can still get to the super call + EmbeddedAnsible.disable + super + rescue SystemExit + _log.info("#{log_prefix} SystemExit received, exiting monitoring Thread") + Thread.exit + end + + def update_embedded_ansible_manager + ansible = ManageIQ::Providers::EmbeddedAnsible::AutomationManager.first_or_initialize + server = MiqServer.my_server(true) + ansible.default_endpoint.url = URI::HTTPS.build(:host => server.hostname, :path => "/ansibleapi/v1") + ansible.name = "Embedded Ansible" + ansible.zone = server.zone + ansible.save! + end + + # Base class methods we override since we don't have a separate process. We might want to make these opt-in features in the base class that this subclass can choose to opt-out. + def set_process_title; end + def set_connection_pool_size; end + def message_sync_active_roles(*_args); end + def message_sync_config(*_args); end +end diff --git a/app/models/miq_server/worker_management/monitor/class_names.rb b/app/models/miq_server/worker_management/monitor/class_names.rb index 9ec0f26ad46..fed55eae327 100644 --- a/app/models/miq_server/worker_management/monitor/class_names.rb +++ b/app/models/miq_server/worker_management/monitor/class_names.rb @@ -57,6 +57,7 @@ module MiqServer::WorkerManagement::Monitor::ClassNames ManageIQ::Providers::StorageManager::CinderManager::EventCatcher ManageIQ::Providers::Vmware::InfraManager::EventCatcher ManageIQ::Providers::Vmware::CloudManager::EventCatcher + EmbeddedAnsibleWorker MiqEventHandler MiqGenericWorker MiqNetappRefreshWorker @@ -86,6 +87,7 @@ module MiqServer::WorkerManagement::Monitor::ClassNames ManageIQ::Providers::Openstack::CloudManager::MetricsCollectorWorker ManageIQ::Providers::Openstack::NetworkManager::MetricsCollectorWorker ManageIQ::Providers::Openstack::InfraManager::MetricsCollectorWorker + EmbeddedAnsibleWorker MiqReportingWorker MiqSmartProxyWorker MiqGenericWorker diff --git a/app/models/miq_worker.rb b/app/models/miq_worker.rb index 5cb7270c109..f40561d41e7 100644 --- a/app/models/miq_worker.rb +++ b/app/models/miq_worker.rb @@ -399,6 +399,10 @@ def is_stopped? STATUSES_STOPPED.include?(status) end + def started? + STATUS_STARTED == status + end + def actually_running? MiqProcess.is_worker?(pid) end diff --git a/config/settings.yml b/config/settings.yml index 75fc7f77053..9fb97fdab07 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -1192,6 +1192,9 @@ :poll_method: :normal :restart_interval: 0.hours :starting_timeout: 10.minutes + :embedded_ansible_worker: + :poll: 10.seconds + :memory_threshold: 0.megabytes :ems_refresh_core_worker: :poll: 1.seconds :nice_delta: 1 diff --git a/spec/factories/miq_worker.rb b/spec/factories/miq_worker.rb index 5cfb615ee4b..7f2f9651654 100644 --- a/spec/factories/miq_worker.rb +++ b/spec/factories/miq_worker.rb @@ -16,4 +16,8 @@ factory :ems_refresh_worker_amazon, :parent => :miq_ems_refresh_worker, :class => "ManageIQ::Providers::Amazon::CloudManager::RefreshWorker" + + factory :embedded_ansible_worker, + :parent => :miq_worker, + :class => "EmbeddedAnsibleWorker" end diff --git a/spec/models/embedded_ansible_worker/runner_spec.rb b/spec/models/embedded_ansible_worker/runner_spec.rb new file mode 100644 index 00000000000..a73d6537301 --- /dev/null +++ b/spec/models/embedded_ansible_worker/runner_spec.rb @@ -0,0 +1,40 @@ +describe EmbeddedAnsibleWorker::Runner do + context ".new" do + let(:miq_server) { + s = EvmSpecHelper.create_guid_miq_server_zone[1] + s.update(:hostname => "fancyserver") + s + } + let(:worker_guid) { MiqUUID.new_guid } + let(:worker) { FactoryGirl.create(:embedded_ansible_worker, :guid => worker_guid, :miq_server_id => miq_server.id) } + let(:runner) { + worker + allow_any_instance_of(described_class).to receive(:worker_initialization) + described_class.new(:guid => worker_guid) + } + + context "#update_embedded_ansible_manager" do + it "creates initial" do + runner.update_embedded_ansible_manager + + ansible = ManageIQ::Providers::EmbeddedAnsible::AutomationManager.first + expect(ansible.zone).to eq(miq_server.zone) + expect(ansible.default_endpoint.url).to eq("https://fancyserver/ansibleapi/v1") + end + + it "updates existing" do + runner.update_embedded_ansible_manager + new_zone = FactoryGirl.create(:zone) + miq_server.update(:hostname => "boringserver", :zone => new_zone) + + runner.update_embedded_ansible_manager + expect(ManageIQ::Providers::EmbeddedAnsible::AutomationManager.count).to eq(1) + + ansible = ManageIQ::Providers::EmbeddedAnsible::AutomationManager.first + expect(ansible.zone).to eq(new_zone) + expect(ansible.default_endpoint.url).to eq("https://boringserver/ansibleapi/v1") + end + end + end +end +