From e3150b9c9fb42b64342c94a638849d269071f7f2 Mon Sep 17 00:00:00 2001 From: Kenneth Worden Date: Thu, 6 Jan 2022 14:53:32 -0500 Subject: [PATCH 1/2] Fix bug during cluster health check --- eksrollup/lib/aws.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/eksrollup/lib/aws.py b/eksrollup/lib/aws.py index 4880c8c..6f3d39f 100644 --- a/eksrollup/lib/aws.py +++ b/eksrollup/lib/aws.py @@ -4,6 +4,7 @@ import requests from .logger import logger from eksrollup.config import app_config +from .k8s import get_k8s_nodes, get_node_by_instance_id client = boto3.client('autoscaling') ec2_client = boto3.client('ec2') @@ -420,16 +421,28 @@ def get_asg_tag(tags, tag_name): return result -def count_all_cluster_instances(cluster_name, predictive=False): +def count_all_cluster_instances(cluster_name, predictive=False, exclude_node_label_keys=app_config["EXCLUDE_NODE_LABEL_KEYS"]): """ Returns the total number of ec2 instances in a k8s cluster """ + + # Get the K8s nodes on the cluster, while excluding the relevant + k8s_nodes = get_k8s_nodes(exclude_node_label_keys) + count = 0 asgs = get_all_asgs(cluster_name) for asg in asgs: + instances = asg['Instances'] if predictive: count += asg['DesiredCapacity'] else: - count += len(asg['Instances']) + # Use the get_node_by_instance_id() function as it only returns the node if it is not excluded by K8s labels + for instance in instances: + instance_id = instance['InstanceId'] + try: + get_node_by_instance_id(k8s_nodes, instance_id) + count += 1 + except: + logger.info("Skipping instance {}".format(instance_id)) logger.info("{} asg instance count in cluster is: {}. K8s node count should match this number".format("*** Predicted" if predictive else "Current", count)) return count From c1837405a4f49a02757fe6da2b09baa20cd1edf8 Mon Sep 17 00:00:00 2001 From: Kenneth Worden Date: Thu, 6 Jan 2022 14:56:38 -0500 Subject: [PATCH 2/2] Fix comment --- eksrollup/lib/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eksrollup/lib/aws.py b/eksrollup/lib/aws.py index 6f3d39f..e00f67c 100644 --- a/eksrollup/lib/aws.py +++ b/eksrollup/lib/aws.py @@ -426,7 +426,7 @@ def count_all_cluster_instances(cluster_name, predictive=False, exclude_node_lab Returns the total number of ec2 instances in a k8s cluster """ - # Get the K8s nodes on the cluster, while excluding the relevant + # Get the K8s nodes on the cluster, while excluding nodes with certain label keys k8s_nodes = get_k8s_nodes(exclude_node_label_keys) count = 0