Skip to content

Commit

Permalink
v1.1.1 code
Browse files Browse the repository at this point in the history
  • Loading branch information
@Cherrett
Cherrett committed May 17, 2022
1 parent f720bf2 commit aece40e
Show file tree
Hide file tree
Showing 70 changed files with 7,521 additions and 9,226 deletions.
42 changes: 34 additions & 8 deletions alerter/run_alerter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
COSMOS_ALERTERS_MANAGER_NAME)
from src.utils.constants.rabbitmq import (
ALERT_ROUTER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE,
SYS_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME,
SYS_ALERTERS_MAN_CONFIGS_QUEUE_NAME,
CHANNELS_MANAGER_CONFIGS_QUEUE_NAME, GH_MON_MAN_CONFIGS_QUEUE_NAME,
DH_MON_MAN_CONFIGS_QUEUE_NAME, SYS_MON_MAN_CONFIGS_QUEUE_NAME,
NODE_MON_MAN_CONFIGS_QUEUE_NAME, EVM_NODES_CONFIGS_ROUTING_KEY_CHAINS,
Expand All @@ -63,7 +63,8 @@
COSMOS_NODE_ALERTER_INPUT_CONFIGS_QUEUE_NAME,
MONITORABLE_STORE_INPUT_QUEUE_NAME, MONITORABLE_EXCHANGE,
MONITORABLE_STORE_INPUT_ROUTING_KEY,
COSMOS_NETWORK_ALERTER_INPUT_CONFIGS_QUEUE_NAME)
COSMOS_NETWORK_ALERTER_INPUT_CONFIGS_QUEUE_NAME,
SYSTEM_ALERTER_INPUT_CONFIGS_QUEUE_NAME)
from src.utils.constants.starters import (
RE_INITIALISE_SLEEPING_PERIOD, RESTART_SLEEPING_PERIOD,
)
Expand Down Expand Up @@ -922,25 +923,50 @@ def _initialise_and_declare_config_queues() -> None:

# System Alerters Manager queues
log_and_print("Creating queue '{}'".format(
SYS_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME), dummy_logger)
rabbitmq.queue_declare(SYS_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME,
SYS_ALERTERS_MAN_CONFIGS_QUEUE_NAME), dummy_logger)
rabbitmq.queue_declare(SYS_ALERTERS_MAN_CONFIGS_QUEUE_NAME,
False, True, False, False)
log_and_print(
"Binding queue '{}' to '{}' exchange with routing "
"key {}.".format(SYS_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME,
"key {}.".format(SYS_ALERTERS_MAN_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_CHAIN),
dummy_logger)
rabbitmq.queue_bind(SYS_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME,
rabbitmq.queue_bind(SYS_ALERTERS_MAN_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_CHAIN)
log_and_print(
"Binding queue '{}' to '{}' exchange with routing "
"key {}.".format(SYS_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME,
"key {}.".format(SYS_ALERTERS_MAN_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_GEN),
dummy_logger)
rabbitmq.queue_bind(SYS_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME,
rabbitmq.queue_bind(SYS_ALERTERS_MAN_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_GEN)

# System Alerter queues
log_and_print("Creating queue '{}'".format(
SYSTEM_ALERTER_INPUT_CONFIGS_QUEUE_NAME), dummy_logger)
rabbitmq.queue_declare(SYSTEM_ALERTER_INPUT_CONFIGS_QUEUE_NAME,
False, True, False, False)
log_and_print(
"Binding queue '{}' to '{}' exchange with routing "
"key {}.".format(SYSTEM_ALERTER_INPUT_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_CHAIN),
dummy_logger)
rabbitmq.queue_bind(SYSTEM_ALERTER_INPUT_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_CHAIN)

log_and_print(
"Binding queue '{}' to '{}' exchange with routing "
"key {}.".format(SYSTEM_ALERTER_INPUT_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_GEN),
dummy_logger)
rabbitmq.queue_bind(SYSTEM_ALERTER_INPUT_CONFIGS_QUEUE_NAME,
CONFIG_EXCHANGE,
ALERTS_CONFIGS_ROUTING_KEY_GEN)

Expand Down
6 changes: 3 additions & 3 deletions alerter/src/alerter/alert_code/node/chainlink_alert_code.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ class ChainlinkNodeAlertCode(AlertCode):
NoOfUnconfirmedTxsDecreasedBelowThresholdAlert = 'cl_node_alert_10'
TotalErroredJobRunsIncreasedAboveThresholdAlert = 'cl_node_alert_11'
TotalErroredJobRunsDecreasedBelowThresholdAlert = 'cl_node_alert_12'
EthBalanceIncreasedAboveThresholdAlert = 'cl_node_alert_13'
EthBalanceDecreasedBelowThresholdAlert = 'cl_node_alert_14'
EthBalanceToppedUpAlert = 'cl_node_alert_15'
BalanceIncreasedAboveThresholdAlert = 'cl_node_alert_13'
BalanceDecreasedBelowThresholdAlert = 'cl_node_alert_14'
BalanceToppedUpAlert = 'cl_node_alert_15'
InvalidUrlAlert = 'cl_node_alert_16'
ValidUrlAlert = 'cl_node_alert_17'
PrometheusSourceIsDownAlert = 'cl_node_alert_18'
Expand Down
65 changes: 28 additions & 37 deletions alerter/src/alerter/alerter_starters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@
from src.alerter.alerters.node.cosmos import CosmosNodeAlerter
from src.alerter.alerters.node.evm import EVMNodeAlerter
from src.alerter.alerters.system import SystemAlerter
from src.configs.alerts.system import SystemAlertsConfig
from src.configs.factory.alerts.chainlink_alerts import (
ChainlinkNodeAlertsConfigsFactory, ChainlinkContractAlertsConfigsFactory)
from src.configs.factory.alerts.cosmos_alerts import (
CosmosNodeAlertsConfigsFactory, CosmosNetworkAlertsConfigsFactory)
from src.configs.factory.alerts.evm_alerts import EVMNodeAlertsConfigsFactory
from src.configs.factory.alerts.system_alerts import SystemAlertsConfigsFactory
from src.message_broker.rabbitmq import RabbitMQApi
from src.utils.constants.names import (
SYSTEM_ALERTER_NAME_TEMPLATE, GITHUB_ALERTER_NAME, DOCKERHUB_ALERTER_NAME,
SYSTEM_ALERTER_NAME, GITHUB_ALERTER_NAME, DOCKERHUB_ALERTER_NAME,
CHAINLINK_NODE_ALERTER_NAME, CHAINLINK_CONTRACT_ALERTER_NAME,
EVM_NODE_ALERTER_NAME, COSMOS_NODE_ALERTER_NAME,
COSMOS_NETWORK_ALERTER_NAME)
Expand Down Expand Up @@ -56,10 +56,11 @@ def _initialise_alerter_logger(alerter_display_name: str,
return alerter_logger


def _initialise_system_alerter(system_alerts_config: SystemAlertsConfig,
chain: str) -> SystemAlerter:
def _initialise_system_alerter(
system_alerts_configs_factory: SystemAlertsConfigsFactory
) -> SystemAlerter:
# Alerter display name based on system
alerter_display_name = SYSTEM_ALERTER_NAME_TEMPLATE.format(chain)
alerter_display_name = SYSTEM_ALERTER_NAME

system_alerter_logger = _initialise_alerter_logger(alerter_display_name,
SystemAlerter.__name__)
Expand All @@ -70,12 +71,11 @@ def _initialise_system_alerter(system_alerts_config: SystemAlertsConfig,
rabbitmq = RabbitMQApi(
logger=system_alerter_logger.getChild(RabbitMQApi.__name__),
host=RABBIT_IP)
system_alerter = SystemAlerter(alerter_display_name,
system_alerts_config,
system_alerter_logger,
rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE
)
system_alerter = SystemAlerter(
alerter_display_name, system_alerter_logger,
system_alerts_configs_factory, rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE
)
log_and_print("Successfully initialised {}".format(
alerter_display_name), system_alerter_logger)
break
Expand All @@ -100,11 +100,10 @@ def _initialise_github_alerter() -> GithubAlerter:
rabbitmq = RabbitMQApi(
logger=github_alerter_logger.getChild(RabbitMQApi.__name__),
host=RABBIT_IP)
github_alerter = GithubAlerter(alerter_display_name,
github_alerter_logger,
rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE
)
github_alerter = GithubAlerter(
alerter_display_name, github_alerter_logger, rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE
)
log_and_print("Successfully initialised {}".format(
alerter_display_name), github_alerter_logger)
break
Expand All @@ -129,11 +128,10 @@ def _initialise_dockerhub_alerter() -> DockerhubAlerter:
rabbitmq = RabbitMQApi(
logger=dockerhub_alerter_logger.getChild(RabbitMQApi.__name__),
host=RABBIT_IP)
dockerhub_alerter = DockerhubAlerter(alerter_display_name,
dockerhub_alerter_logger,
rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE
)
dockerhub_alerter = DockerhubAlerter(
alerter_display_name, dockerhub_alerter_logger, rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE
)
log_and_print("Successfully initialised {}".format(
alerter_display_name), dockerhub_alerter_logger)
break
Expand Down Expand Up @@ -161,10 +159,8 @@ def _initialise_chainlink_node_alerter(
logger=chainlink_alerter_logger.getChild(RabbitMQApi.__name__),
host=RABBIT_IP)
chainlink_alerter = ChainlinkNodeAlerter(
alerter_display_name,
chainlink_alerter_logger,
rabbitmq,
chainlink_alerts_configs_factory,
alerter_display_name, chainlink_alerter_logger,
rabbitmq, chainlink_alerts_configs_factory,
ALERTER_PUBLISHING_QUEUE_SIZE
)
log_and_print("Successfully initialised {}".format(
Expand Down Expand Up @@ -194,11 +190,8 @@ def _initialise_chainlink_contract_alerter(
logger=chainlink_alerter_logger.getChild(RabbitMQApi.__name__),
host=RABBIT_IP)
chainlink_alerter = ChainlinkContractAlerter(
alerter_display_name,
chainlink_alerter_logger,
rabbitmq,
chainlink_alerts_configs_factory,
ALERTER_PUBLISHING_QUEUE_SIZE
alerter_display_name, chainlink_alerter_logger, rabbitmq,
chainlink_alerts_configs_factory, ALERTER_PUBLISHING_QUEUE_SIZE
)
log_and_print("Successfully initialised {}".format(
alerter_display_name), chainlink_alerter_logger)
Expand Down Expand Up @@ -227,10 +220,8 @@ def _initialise_evm_node_alerter(
logger=evm_node_alerter_logger.getChild(RabbitMQApi.__name__),
host=RABBIT_IP)
evm_node_alerter = EVMNodeAlerter(
alerter_display_name,
evm_node_alerter_logger,
evm_alerts_configs_factory,
rabbitmq,
alerter_display_name, evm_node_alerter_logger,
evm_alerts_configs_factory, rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE
)
log_and_print("Successfully initialised {}".format(
Expand Down Expand Up @@ -305,9 +296,9 @@ def _initialise_cosmos_network_alerter(
return cosmos_alerter


def start_system_alerter(system_alerts_config: SystemAlertsConfig,
chain: str) -> None:
system_alerter = _initialise_system_alerter(system_alerts_config, chain)
def start_system_alerter(
system_alerts_configs_factory: SystemAlertsConfigsFactory) -> None:
system_alerter = _initialise_system_alerter(system_alerts_configs_factory)
start_alerter(system_alerter)


Expand Down
4 changes: 2 additions & 2 deletions alerter/src/alerter/alerters/contract/chainlink.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,8 @@ def _process_configs(self, method: pika.spec.Basic.Deliver,

try:
# Checking if the configuration is empty. If it is empty, remove the
# stored config (if it exists), and if it not add it to the list of
# configurations.
# stored config (if it exists), and if it is not, add it to the
# list of configurations.
if bool(sent_configs):
self.alerts_configs_factory.add_new_config(chain, sent_configs)

Expand Down
33 changes: 17 additions & 16 deletions alerter/src/alerter/alerters/node/chainlink.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,31 +274,32 @@ def _process_prometheus_result(self, prom_data: Dict,
MetricCode.TotalErroredJobRunsThreshold.value,
meta_data['node_name'], meta_data['last_monitored']
)
if str_to_bool(configs.eth_balance_amount['enabled']):
current = data['eth_balance_info']['current']
sub_config = configs.eth_balance_amount
if str_to_bool(configs.balance_amount['enabled']):
current = data['balance_info']['current']
sub_config = configs.balance_amount
if current != {}:
self.alerting_factory.classify_thresholded_alert_reverse(
current['balance'], sub_config,
cl_alerts.EthBalanceIncreasedAboveThresholdAlert,
cl_alerts.EthBalanceDecreasedBelowThresholdAlert,
self.alerting_factory. \
classify_thresholded_alert_reverse_chainlink_node(
current['balance'], sub_config, current['symbol'],
cl_alerts.BalanceIncreasedAboveThresholdAlert,
cl_alerts.BalanceDecreasedBelowThresholdAlert,
data_for_alerting, meta_data['node_parent_id'],
meta_data['node_id'],
MetricCode.EthBalanceThreshold.value,
MetricCode.BalanceThreshold.value,
meta_data['node_name'], meta_data['last_monitored']
)
if str_to_bool(configs.eth_balance_amount_increase['enabled']):
current = data['eth_balance_info']['current']
previous = data['eth_balance_info']['previous']
sub_config = configs.eth_balance_amount_increase
if str_to_bool(configs.balance_amount_increase['enabled']):
current = data['balance_info']['current']
previous = data['balance_info']['previous']
sub_config = configs.balance_amount_increase
if current != {} and previous != {}:
increase = current['balance'] - previous['balance']
self.alerting_factory.classify_conditional_alert(
cl_alerts.EthBalanceToppedUpAlert,
cl_alerts.BalanceToppedUpAlert,
self._greater_than_condition_function,
[current['balance'], previous['balance']], [
meta_data['node_name'], current['balance'],
increase, sub_config['severity'],
increase, current['symbol'], sub_config['severity'],
meta_data['last_monitored'],
meta_data['node_parent_id'], meta_data['node_id']
], data_for_alerting
Expand Down Expand Up @@ -555,8 +556,8 @@ def _process_configs(self, method: pika.spec.Basic.Deliver,

try:
# Checking if the configuration is empty. If it is empty, remove the
# stored config (if it exists), and if it not add it to the list of
# configurations.
# stored config (if it exists), and if it is not, add it to the
# list of configurations.
if bool(sent_configs):
self.alerts_configs_factory.add_new_config(chain, sent_configs)

Expand Down
9 changes: 4 additions & 5 deletions alerter/src/alerter/alerters/node/evm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,15 +313,15 @@ def _process_configs(self, method: pika.spec.Basic.Deliver,

try:
# Checking if the configuration is empty. If it is empty, remove the
# stored config (if it exists), and if it not add it to the list of
# configurations.
# stored config (if it exists), and if it is not, add it to the
# list of configurations.
if bool(sent_configs):
self.alerts_configs_factory.add_new_config(chain, sent_configs)

# We must reset the state since some thresholds might have
# changed. A node's state will be recreated in the next
# monitoring round automatically. Note we are sure that a
# parent_id is to be returned, as we have just added the config
# parent_id is to be returned, as we have just added the config.
parent_id = self.alerts_configs_factory.get_parent_id(chain)
self.alerting_factory.remove_chain_alerting_state(parent_id)
else:
Expand All @@ -332,8 +332,7 @@ def _process_configs(self, method: pika.spec.Basic.Deliver,
# no storing took place, therefore in that case do nothing.
parent_id = self.alerts_configs_factory.get_parent_id(chain)
if parent_id:
self.alerting_factory.remove_chain_alerting_state(
parent_id)
self.alerting_factory.remove_chain_alerting_state(parent_id)
self.alerts_configs_factory.remove_config(chain)
except Exception as e:
# Otherwise log and reject the message
Expand Down
Loading

0 comments on commit aece40e

Please sign in to comment.