Merge pull request #59 from GSA/add-more-templates

Solr Restarts - add missing template files

.github/workflows/test_ecs_leader_follower_efs.yml

@@ -26,9 +26,6 @@ jobs:
         shell: bash
 
     steps:
-    - name: Create KinD cluster
-      uses: helm/[email protected]
-
     - name: Check out repository
       uses: actions/checkout@v2
       with:

.github/workflows/test_ecs_leader_follower_efs_ephemeral.yml

@@ -26,9 +26,6 @@ jobs:
         shell: bash
 
     steps:
-    - name: Create KinD cluster
-      uses: helm/[email protected]
-
     - name: Check out repository
       uses: actions/checkout@v2
       with:

.github/workflows/test_ecs_leader_follower_ephemeral.yml

@@ -26,9 +26,6 @@ jobs:
         shell: bash
 
     steps:
-    - name: Create KinD cluster
-      uses: helm/[email protected]
-
     - name: Check out repository
       uses: actions/checkout@v2
       with:

.github/workflows/test_ecs_standalone_efs.yml

@@ -26,9 +26,6 @@ jobs:
         shell: bash
 
     steps:
-    - name: Create KinD cluster
-      uses: helm/[email protected]
-
     - name: Check out repository
       uses: actions/checkout@v2
       with:

.github/workflows/test_ecs_standalone_ephemeral.yml

@@ -26,9 +26,6 @@ jobs:
         shell: bash
 
     steps:
-    - name: Create KinD cluster
-      uses: helm/[email protected]
-
     - name: Check out repository
       uses: actions/checkout@v2
       with:

Makefile

@@ -2,7 +2,7 @@
 .DEFAULT_GOAL := help
 
 DOCKER_OPTS=--rm -v $(PWD):/brokerpak -w /brokerpak
-CSB=ghcr.io/gsa/cloud-service-broker:v0.10.0gsa
+CSB=ghcr.io/gsa/cloud-service-broker:v0.10.0gsa-zip
 SECURITY_USER_NAME := $(or $(SECURITY_USER_NAME), user)
 SECURITY_USER_PASSWORD := $(or $(SECURITY_USER_PASSWORD), pass)
 

bin/utils.go

@@ -24,7 +24,7 @@ var excludeRegions = []string{}
 // var resourceTypes = []string{"ec2", "vpc", "efs", "ecsserv", "ecscluster", "cloudwatch-loggroup", "eip", "nat-gateway", "elbv2"}
 // var resultTypes = []string{"elbv2", "nat-gateway", "ec2", "eip", "ecsserv", "ecscluster", "cloudwatch-loggroup", "vpc", "efs"}
 var resourceTypes = []string{"vpc"}
-var resultTypes = []string{"vpc"}
+var resultTypes = resourceTypes
 var excludeResourceTypes = []string{}
 // excludeAfter is parsed identically to the --older-than flag
 var excludeAfter = time.Now()

solr-on-ecs.yml

@@ -191,13 +191,16 @@ provision:
     required: true
   template_refs:
     admin: terraform/ecs/provision/admin.tf
+    app-template: terraform/ecs/provision/app_template.tf
+    cloudwatch: terraform/ecs/provision/cloudwatch.tf
     dns: terraform/ecs/provision/dns.tf
     private-dns: terraform/ecs/provision/private-dns.tf
     leader: terraform/ecs/provision/leader.tf
     efs: terraform/ecs/provision/efs-leader.tf
     iam: terraform/ecs/provision/iam.tf
     lb: terraform/ecs/provision/lb-leader.tf
     logging: terraform/ecs/provision/logging.tf
+    restarts: terraform/ecs/provision/restarts.tf
     ssl: terraform/ecs/provision/ssl.tf
     vpc: terraform/ecs/provision/vpc.tf
     outputs: terraform/ecs/provision/outputs.tf

terraform/ecs/provision/app_template.tf

@@ -0,0 +1,104 @@
+
+locals {
+  app_template = <<-PYTHON
+    import boto3
+    import json
+    print('Loading function')
+
+    def handler(event, context):
+        print("Full event: " + json.dumps(event, indent=2))
+        # Parse Message
+        message = event['Records'][0]['Sns']['Message']
+        print("From SNS: " + message)
+        message_json = json.loads(message)
+
+        # Parse Alarm State
+        state = message_json['NewStateValue']
+
+        # Parse/Restart ECS Service
+        service_dimensions = identifyCluster(message_json)
+        if state == 'ALARM':
+            restartSolrECS(message_json, service_dimensions['ClusterName'], service_dimensions['ServiceName'])
+            notifySlack(message_json, service_dimensions['ClusterName'], service_dimensions['ServiceName'])
+        elif state == 'OK':
+            notifySlack(message_json, service_dimensions['ClusterName'], service_dimensions['ServiceName'])
+
+        return message
+
+
+    def identifyCluster(event_info):
+        service_dimensions = {}
+        for dim in event_info['Trigger']['Dimensions']:
+            service_dimensions[dim['name']] = dim['value']
+        return service_dimensions
+
+
+    def restartSolrECS(message_json, cluster_name, service_name):
+        '''
+        Reference: https://github.com/s7anley/aws-ecs-service-stop-lambda/blob/master/main.py
+        '''
+        service_region = message_json['AlarmArn'].split(':')[3]
+
+        client = boto3.client("ecs", region_name=service_region)
+        response = client.list_tasks(
+            cluster=cluster_name,
+            family="%s-service" % (service_name))
+        tasks = response.get('taskArns', [])
+        print("Service is running {0} underlying tasks".format(len(tasks)))
+
+        for task in tasks:
+            print("Stopping tasks {0}".format(tasks))
+            client.stop_task(cluster=cluster_name, task=task)
+
+        print("Completed service restart")
+
+
+    def notifySlack(event_info, cluster, service):
+        from slack_sdk.webhook import WebhookClient
+        webhook = WebhookClient("<slack-notification-url>")
+
+        important_data = ["Alarm Name", "New State Value", "New State Reason", "State Change Time"]
+        emoji = "😨" if event_info["NewStateValue"] == "ALARM" else "😐"
+
+        response = webhook.send(blocks=[
+            {
+                "type": "divider"
+            },
+            {
+                "type": "context",
+                "elements": [
+                    {
+                        "type": "image",
+                        "image_url": "https://s3-us-gov-west-1.amazonaws.com/cg-0817d6e3-93c4-4de8-8b32-da6919464e61/solr.png",
+                        "alt_text": "Solr Icon"
+                    },
+                    {
+                        "type": "mrkdwn",
+                        "text": ":::ALERT::: *%s/%s* has experienced an event. " % (cluster, service) + emoji
+                    }
+                ]
+            },
+            {
+                "type": "divider"
+            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": "".join(["*%s*:\n%s\n\n" % (key, event_info[key.replace(" ", "")]) for key in important_data])
+                },
+                "accessory": {
+                    "type": "image",
+    		"image_url": "https://s3-us-gov-west-1.amazonaws.com/cg-0817d6e3-93c4-4de8-8b32-da6919464e61/solr.png",
+                    "alt_text": "Solr Alert"
+                }
+            },
+            {
+                "type": "divider"
+            }
+        ])
+
+        assert response.status_code == 200
+        assert response.body == "ok"
+    PYTHON
+}

terraform/ecs/provision/restarts.tf

@@ -6,7 +6,7 @@ resource "aws_sns_topic_subscription" "solr_memory_lambda_target" {
 }
 
 resource "aws_iam_role" "iam_for_lambda" {
-  name = "iam_for_solr_restarts"
+  name = "iam_for_solr_${local.lb_name}_restarts"
 
   assume_role_policy = <<-EOF
   {
@@ -77,13 +77,13 @@ data "aws_secretsmanager_secret_version" "slackNotificationUrl" {
 
 resource "local_file" "app" {
   count = var.slackNotification ? 1 : 0
-  content  = replace(file("${path.module}/app_template.py"), "<slack-notification-url>", jsondecode(data.aws_secretsmanager_secret_version.slackNotificationUrl[0].secret_string)["slackNotificationUrl"])
+  content  = replace(local.app_template, "<slack-notification-url>", jsondecode(data.aws_secretsmanager_secret_version.slackNotificationUrl[0].secret_string)["slackNotificationUrl"])
   filename = "${path.module}/app.py"
 }
 
 resource "local_file" "app_no_slack" {
   count = var.slackNotification == false ? 1 : 0
-  content  = replace(file("${path.module}/app_template.py"), "notifySlack(message_json, service_dimensions['ClusterName'], service_dimensions['ServiceName'])", "")
+  content  = replace(local.app_template, "notifySlack(message_json, service_dimensions['ClusterName'], service_dimensions['ServiceName'])", "")
   filename = "${path.module}/app.py"
 }