Make sure what is in AWS is what is in git

aws/src/main/python/beam_lambda/lambda_function.py

@@ -74,36 +74,14 @@
   - crontab -l
   - echo "notification scheduled..."
   - git fetch
-  - echo "git checkout ..."
+  - 'echo "git checkout: $(date)"'
   - GIT_LFS_SKIP_SMUDGE=1 sudo git checkout $BRANCH
   - sudo git pull
   - sudo git lfs pull
   - echo "git checkout -qf ..."
   - GIT_LFS_SKIP_SMUDGE=1 sudo git checkout -qf $COMMIT
-  - echo "preparing for python analysis"
-  - sudo dpkg --configure -a
-  - sudo dpkg --remove --force-remove-reinstreq  unattended-upgrades
-  - sudo apt-get install unattended-upgrades
-  - sudo dpkg --configure -a
-  - sudo apt update
-  - sudo apt install npm -y
-  - sudo apt install nodejs-legacy -y
-  - sudo apt install python-pip -y
-  - pip install --upgrade pip
-  - sudo pip install pandas
-  - sudo pip install plotly
-  - sudo pip install psutil requests
-  - sudo npm cache clean -f
-  - sudo npm install -g n
-  - sudo n stable
-  - sudo npm install -g npm
-  - sudo apt-get install curl
-  - curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
-  - sudo apt-get install nodejs -y
-  - sudo apt-get install libgtkextra-dev libgconf2-dev libnss3 libasound2 libxtst-dev -y
-  - sudo npm install -g [email protected] orca --unsafe-perm=true --alow-root -y
-  - sudo apt-get install xvfb -y
-  - echo "gradlew assemble ..."
+
+  - 'echo "gradlew assemble: $(date)"'
   - ./gradlew assemble
   - echo "looping config ..."
   - export MAXRAM=$MAX_RAM
@@ -192,30 +170,30 @@ def validate(name):
 
 def deploy(script, instance_type, region_prefix, shutdown_behaviour, instance_name, volume_size):
     res = ec2.run_instances(BlockDeviceMappings=[
-                                {
-                                    'DeviceName': '/dev/sda1',
-                                    'Ebs': {
-                                        'VolumeSize': volume_size,
-                                        'VolumeType': 'gp2'
-                                    }
-                                }
-                            ],
-                            ImageId=os.environ[region_prefix + 'IMAGE_ID'],
-                            InstanceType=instance_type,
-                            UserData=script,
-                            KeyName=os.environ[region_prefix + 'KEY_NAME'],
-                            MinCount=1,
-                            MaxCount=1,
-                            SecurityGroupIds=[os.environ[region_prefix + 'SECURITY_GROUP']],
-                            IamInstanceProfile={'Name': os.environ['IAM_ROLE'] },
-                            InstanceInitiatedShutdownBehavior=shutdown_behaviour,
-                            TagSpecifications=[ {
-                                'ResourceType': 'instance',
-                                'Tags': [ {
-                                    'Key': 'Name',
-                                    'Value': instance_name
-                                } ]
-                            } ])
+        {
+            'DeviceName': '/dev/sda1',
+            'Ebs': {
+                'VolumeSize': volume_size,
+                'VolumeType': 'gp2'
+            }
+        }
+    ],
+        ImageId=os.environ[region_prefix + 'IMAGE_ID'],
+        InstanceType=instance_type,
+        UserData=script,
+        KeyName=os.environ[region_prefix + 'KEY_NAME'],
+        MinCount=1,
+        MaxCount=1,
+        SecurityGroupIds=[os.environ[region_prefix + 'SECURITY_GROUP']],
+        IamInstanceProfile={'Name': os.environ['IAM_ROLE'] },
+        InstanceInitiatedShutdownBehavior=shutdown_behaviour,
+        TagSpecifications=[ {
+            'ResourceType': 'instance',
+            'Tags': [ {
+                'Key': 'Name',
+                'Value': instance_name
+            } ]
+        } ])
     return res['Instances'][0]['InstanceId']
 
 def get_dns(instance_id):

aws/src/main/python/cloneAMI/lambda_function.py

@@ -0,0 +1,24 @@
+# coding=utf-8
+import time
+import json
+import boto3
+import logging
+from botocore.errorfactory import ClientError
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+def lambda_handler(event, context):
+    ami_id = event.get('ami_id')
+    image_name = event.get('ami_name')
+    region = event.get('region')
+
+    return copy_ami(image_name, ami_id, region)
+
+def copy_ami(image_name, image_id, region):
+    ec2 = boto3.client('ec2',region_name=region)
+    res = ec2.copy_image(Name=image_name,
+                         SourceImageId=image_id,
+                         SourceRegion='us-east-2')
+    logger.info('ami [' + image_id + '] coppied to region ' + region + ' with id ' + res['ImageId'])
+    return res['ImageId']

aws/src/main/python/createAMI/lambda_function.py

@@ -0,0 +1,31 @@
+# coding=utf-8
+import time
+import json
+import boto3
+import logging
+from botocore.errorfactory import ClientError
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+def lambda_handler(event, context):
+    instance_id = event.get('instance_id')
+    terminate_instance = event.get('terminate_instance', "false")
+
+    image_name = 'beam-automation-'+time.strftime("%Y-%m-%d-%H%M%S", time.gmtime())
+    ami_id = create_ami(image_name, instance_id, terminate_instance)
+
+    return ami_id + "|" + image_name
+
+
+def create_ami(image_name, instance_id, terminate_instance):
+    ec2 = boto3.client('ec2',region_name='us-east-2')
+    logger.info('creating ami of instance' + instance_id)
+    res = ec2.create_image(InstanceId=instance_id,
+                           Name=image_name, NoReboot=True)
+    logger.info('ami creation started for ' + res['ImageId'])
+    if terminate_instance == "true":
+        logger.info('shutting down instance' + instance_id)
+        ec2.terminate_instances(InstanceIds=[instance_id])
+        logger.info('shutting down complete for instance' + instance_id)
+    return res['ImageId']

aws/src/main/python/updateBeamAMI/lambda_function.py

@@ -2,60 +2,72 @@
 import time
 import json
 import boto3
+import logging
 from botocore.errorfactory import ClientError
 
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
 def lambda_handler(event, context):
     instance_id = event.get('instance_id')
     region_id = event.get('region_id', 'us-east-2')
-    
+
     image_name = 'beam-automation-'+time.strftime("%Y-%m-%d-%H%M%S", time.gmtime())
     image_ids = {}
-
+
+
     image_ids['us-east-2'] = create_ami(image_name, instance_id)
     image_ids['us-east-1'] = copy_ami(image_name, image_ids['us-east-2'], 'us-east-1')
     image_ids['us-west-2'] = copy_ami(image_name, image_ids['us-east-2'], 'us-west-2')
     update_lambda(image_ids)
-    
+
     return json.dumps(image_ids)
 
 
 def create_ami(image_name, instance_id):
     ec2 = boto3.client('ec2',region_name='us-east-2')
+    logger.info('creating ami of instance' + instance_id)
     res = ec2.create_image(InstanceId=instance_id,
-                            Name=image_name)
+                           Name=image_name)
     wait4image(ec2, res['ImageId'])
-    ec2.terminate_instances(InstanceIds=[instance_id]) 
+    logger.info('ami created ' + res['ImageId'])
+    logger.info('shutting down instance' + instance_id)
+    ec2.terminate_instances(InstanceIds=[instance_id])
+    logger.info('shutting down complete for instance' + instance_id)
     return res['ImageId']
-    
+
 def copy_ami(image_name, image_id, region):
     ec2 = boto3.client('ec2',region_name=region)
     res = ec2.copy_image(Name=image_name,
-                    SourceImageId=image_id,
-                    SourceRegion='us-east-2')
+                         SourceImageId=image_id,
+                         SourceRegion='us-east-2')
+    logger.info('ami [' + image_id + '] coppied to region ' + region + ' with id ' + res['ImageId'])
     # wait4image(ec2, res['ImageId'])
     return res['ImageId']
-    
+
 def wait4image(ec2, image_id):
     waiter = ec2.get_waiter('image_available')
     waiter.wait(Filters=[{'Name': 'state', 'Values': ['available']}],
                 ImageIds=[image_id])
-                
+
 def update_lambda(image_ids):
     lm = boto3.client('lambda')
+    logger.info('updateing image ids ' + str(image_ids))
     en_var = lm.get_function_configuration(FunctionName='simulateBeam')['Environment']['Variables']
     en_var.update({
-                    'us_east_2_IMAGE_ID': image_ids['us-east-2'],
-                    'us_east_1_IMAGE_ID': image_ids['us-east-1'],
-                    'us_west_2_IMAGE_ID': image_ids['us-west-2'],
-                })
+        'us_east_2_IMAGE_ID': image_ids['us-east-2'],
+        'us_east_1_IMAGE_ID': image_ids['us-east-1'],
+        'us_west_2_IMAGE_ID': image_ids['us-west-2'],
+    })
     lm.update_function_configuration(
-            FunctionName='simulateBeam',
-            Environment={
-                'Variables': en_var
-            }
-        )
+        FunctionName='simulateBeam',
+        Environment={
+            'Variables': en_var
+        }
+    )
+    logger.info('simulateBeam image ids updated')
+
 
-
 def check_instance_id(instance_ids):
     for reservation in ec2.describe_instances()['Reservations']:
         for instance in reservation['Instances']:

aws/src/main/python/updateDependencies/lambda_function.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 import os
 import boto3
+from botocore.errorfactory import ClientError
 
 initscript = (('''#cloud-config
 runcmd:
@@ -12,20 +13,73 @@
   - sudo git reset origin/HEAD
   - sudo git checkout -- .
   - sudo git clean -df
-  - git fetch
+  - sudo git checkout develop
+  - sudo git pull
+  - sudo git fetch
+  - sudo git fetch --prune
+  - sudo git remote prune origin
+  - echo "-git remote prune origin is done-"
+  - sudo chown -R ubuntu:ubuntu /home/ubuntu/git/beam
   - for bn in $BRANCH
   -  do
   -    echo "-------------------checkout $bn----------------------"
-  -    GIT_LFS_SKIP_SMUDGE=1 sudo git checkout $bn
+  -    sudo GIT_LFS_SKIP_SMUDGE=1 git checkout $bn
+  -    sudo git reset --hard origin/$bn
   -    sudo git pull
   -    sudo git lfs pull
   -  done
+  - sudo chown -R ubuntu:ubuntu /home/ubuntu/git/beam
   - echo "gradlew assemble ..."
   - ./gradlew assemble
   - ./gradlew clean
-  - echo "invoke lambda ..."
-  - sudo aws lambda invoke --invocation-type RequestResponse --function-name updateBeamAMI --region 'us-east-2' --payload '{"instance_id":"'"$(ec2metadata --instance-id)"'","region_id":"us-east-2"}' outputfile.txt
+  - echo "preparing for python analysis"
+  - sudo dpkg --configure -a
+  - sudo dpkg --remove --force-remove-reinstreq  unattended-upgrades
+  - sudo apt-get install unattended-upgrades
+  - sudo dpkg --configure -a
+  - sudo apt update
+  - sudo apt install npm -y
+  - sudo apt install nodejs-legacy -y
+  - sudo apt install python-pip -y
+  - pip install --upgrade pip
+  - sudo pip install pandas
+  - sudo pip install plotly
+  - sudo pip install psutil requests
+  - sudo npm cache clean -f
+  - sudo npm install -g n
+  - sudo n stable
+  - sudo npm install -g npm
+  - sudo apt-get install curl
+  - curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
+  - sudo apt-get install nodejs -y
+  - sudo apt-get install libgtkextra-dev libgconf2-dev libnss3 libasound2 libxtst-dev -y
+  - sudo npm install -g [email protected] orca --unsafe-perm=true --alow-root -y
+  - sudo apt-get install xvfb -y
+  - 'echo resetting git to base: "$(date)"'
+  - sudo git reset --hard 
+  - 'echo fetching the latest: "$(date)"'
+  - sudo git fetch
+  - 'echo current git status: "$(date)"'
+  - sudo git status
+  - 'echo invoke create ami lambda after a 5 minute sleep to let the file system settle..."$(date)"'
+  - sudo sleep 5m
+  - sudo aws lambda invoke --invocation-type RequestResponse --function-name createAMI --region 'us-east-2' --payload '{"instance_id":"'"$(ec2metadata --instance-id)"'","region_id":"us-east-2"}' outputfile.txt
+  - export created_ami_id=$(sed -r 's/"(.*)\|.*/\\1/' outputfile.txt)
+  - export ami_filename=$(sed -r 's/.*\|(.*)"/\\1/' outputfile.txt)
+  - while ! aws ec2 describe-images --image-ids $created_ami_id --region us-east-2 | grep "available"; do echo "Waiting 30 seconds for AMI in us-east-2 $created_ami_id ..."; sleep 30s; done
+  - echo "invoke copy ami lambda to us-east-1 ..."
+  - sudo aws lambda invoke --invocation-type RequestResponse --function-name cloneAMI --region 'us-east-2' --payload '{"ami_id":"'"$created_ami_id"'","ami_name":"'"$ami_filename"'","region":"us-east-1"}' outputfile.txt
+  - export created_ami_id_us_east_1=$(sed -r 's/"(.*)"/\\1/' outputfile.txt)
+  - echo "invoke copy ami lambda to us-west-2 ..."
+  - sudo aws lambda invoke --invocation-type RequestResponse --function-name cloneAMI --region 'us-east-2' --payload '{"ami_id":"'"$created_ami_id"'","ami_name":"'"$ami_filename"'","region":"us-west-2"}' outputfile.txt
+  - export created_ami_id_us_west_2=$(sed -r 's/"(.*)"/\\1/' outputfile.txt)
+  - while ! aws ec2 describe-images --image-ids $created_ami_id_us_east_1 --region us-east-1 | grep "available"; do echo "Waiting 30 seconds for AMI in us-east-1 $created_ami_id_us_east_1 ..."; sleep 30s; done
+  - while ! aws ec2 describe-images --image-ids $created_ami_id_us_west_2 --region us-west-2 | grep "available"; do echo "Waiting 30 seconds for AMI in us-west-2 $created_ami_id_us_west_2 ..."; sleep 30s; done
+  - echo "invoke update simulateBeam lambda ..."
+  - sudo aws lambda invoke --invocation-type RequestResponse --function-name updateSimulateBeamEnvVars --region 'us-east-2' --payload '{"ami_id":"'"$created_ami_id"'","ami_id_us_east_1":"'"$created_ami_id_us_east_1"'","ami_id_us_west_2":"'"$created_ami_id_us_west_2"'"}' outputfile.txt
+  - echo "setting up auto shutdown ..."
   - sudo shutdown -h +$SHUTDOWN_WAIT
+  - echo "shutdown in $SHUTDOWN_WAIT ..."
 '''))
 
 
@@ -61,8 +115,8 @@ def lambda_handler(event, context):
 
     region = 'us-east-2'
     instance_type = os.environ['INSTANCE_TYPE']
-    shutdown_behaviour = 'stop'
-    shutdown_wait = "1"
+    shutdown_behaviour = 'terminate'
+    shutdown_wait = "10"
     runName = 'update-beam-dependencies'
     branches = os.environ['BRANCHES']
 

aws/src/main/python/updateSimulateBeamEnvVars/lambda_function.py

@@ -0,0 +1,36 @@
+# coding=utf-8
+import time
+import json
+import boto3
+import logging
+from botocore.errorfactory import ClientError
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+def lambda_handler(event, context):
+    image_ids = {}
+    image_ids['us-east-2'] = event.get('ami_id')
+    image_ids['us-east-1'] = event.get('ami_id_us_east_1')
+    image_ids['us-west-2'] = event.get('ami_id_us_west_2')
+
+    update_lambda(image_ids)
+
+    return "Update complete"
+
+def update_lambda(image_ids):
+    lm = boto3.client('lambda')
+    logger.info('updating image ids ' + str(image_ids))
+    en_var = lm.get_function_configuration(FunctionName='simulateBeam')['Environment']['Variables']
+    en_var.update({
+        'us_east_2_IMAGE_ID': image_ids['us-east-2'],
+        'us_east_1_IMAGE_ID': image_ids['us-east-1'],
+        'us_west_2_IMAGE_ID': image_ids['us-west-2']
+    })
+    lm.update_function_configuration(
+        FunctionName='simulateBeam',
+        Environment={
+            'Variables': en_var
+        }
+    )
+    logger.info('simulateBeam image ids updated')