Merge pull request #1711 from hotosm/fix/enable-db-logging

3.3.2 Release

.circleci/config.yml

@@ -245,7 +245,7 @@ workflows:
         filters:
           branches:
             only:
-              - master
+              - deployment/hot-tasking-manager
         requires:
           - build
         stack_name: "production"
@@ -272,7 +272,7 @@ workflows:
         filters:
           branches:
             only:
-              - teachosm
+              - deployment/teachosm-tasking-manager
         requires:
           - build
         stack_name: "teachosm"
@@ -299,7 +299,7 @@ workflows:
         filters:
           branches:
             only:
-              - machine-learning
+              - deployment/assisted-tasking-manager
         requires:
           - build
         stack_name: "assisted"

devops/cloudformation/tasking-manager.template.js

@@ -127,9 +127,21 @@ const Resources = {
         AutoScalingGroupName: cf.ref('TaskingManagerASG'),
         PolicyType: 'TargetTrackingScaling',
         TargetTrackingConfiguration: {
-          TargetValue: 85,
+          TargetValue: 500,
           PredefinedMetricSpecification: {
-            PredefinedMetricType: 'ASGAverageCPUUtilization'
+            PredefinedMetricType: 'ALBRequestCountPerTarget',
+            ResourceLabel: cf.join('/', [
+              cf.select(1,
+                cf.split('loadbalancer/',
+                  cf.select(5,
+                    cf.split(':', cf.ref("TaskingManagerLoadBalancer"))
+                  )
+                )
+              ),
+              cf.select(5,
+                cf.split(':', cf.ref("TaskingManagerTargetGroup"))
+              )
+            ])
           }
         },
         Cooldown: 300
@@ -418,6 +430,8 @@ const Resources = {
         AllocatedStorage: cf.ref('DatabaseSize'),
         BackupRetentionPeriod: 10,
         StorageType: 'gp2',
+        DBParameterGroupName: 'tm3-logging-postgres11',
+        EnableCloudwatchLogsExports: ['postgresql'],
         DBInstanceClass: cf.if('IsTaskingManagerProduction', 'db.m3.large', 'db.t2.small'),
         DBSnapshotIdentifier: cf.if('UseASnapshot', cf.ref('DBSnapshot'), cf.noValue),
         VPCSecurityGroups: [cf.importValue(cf.join('-', ['hotosm-network-production', cf.ref('Environment'), 'ec2s-security-group', cf.region]))],

devops/scripts/pg_scripts/README

@@ -0,0 +1,16 @@
+# Shell migration script
+
+These migration scripts allow to restore deleted projects using postgres commands. For now, there are two scripts:
+1. dbtocsv.sh: By setting the required variables, this script reads the defined project ids within the PROJECT_IDS variable and starts saving the required information for each table in a csv file.
+2. restart_table.sh: which destroys and then creates with postgis extension the desired database. However, migration tables are not executed within this script. You need to run `python manage.py db upgrade` yourself everytime the database is restarted.
+
+Tables to migrate and verify if they are necessary to:
+[x] projects 
+[] messages
+[x] project_allowed_users
+[x] project_chat
+[x] project_info
+[x] project_priority_areas
+[x] task_history
+[x] task_invalidation_history
+[x] tasks

devops/scripts/pg_scripts/csv_to_db.sh

@@ -0,0 +1,106 @@
+
+do_dash() {
+   echo "---------------------------------------------------------------"
+}
+
+do_dash; echo "\n Defining variables"
+
+RDS_LIST=( "prod-05-jun" "prod-06-jun" "prod-07-jun" "prod-08-jun" "prod-09-jun"
+           "prod-10-jun" "prod-11-jun" "prod-snapshot-before-12-jun-deploy" "prod-12-jun" )
+FOLDER_PATH=dump/lost_projects
+
+for HOST in "${RDS_LIST[@]}"; do
+  echo $HOST
+  do_dash; echo "Priority areas tables"
+  TABLE=priority_areas
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Projects table"
+  TABLE=projects
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project allowed users"
+  TABLE=project_allowed_users
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Tasks table"
+  TABLE=tasks
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Task history"
+  TABLE=task_history
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Task invalidation history"
+  TABLE=task_invalidation_history
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project priority areas"
+  TABLE=project_priority_areas
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project chat"
+  TABLE=project_chat
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project info"
+  TABLE=project_info
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy $TABLE FROM '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Messages"
+  TABLE=messages
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"CREATE TABLE temp_$TABLE(LIKE messages);"
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+    	"\copy temp_$TABLE FROM '$FILE' DELIMITER ',' CSV HEADER;"
+  psql "host=$TARGET_HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+      	"UPDATE $TABLE SET
+      		task_id=temp_$TABLE.task_id,
+      		project_id=temp_$TABLE.project_id
+      	FROM temp_$TABLE
+      	WHERE
+      		temp_$TABLE.id=$TABLE.id
+      	AND
+      		temp_$TABLE.id IN (SELECT id FROM $TABLE);
+      	INSERT INTO $TABLE SELECT * FROM temp_$TABLE WHERE id NOT IN (SELECT id FROM messages);
+      	DROP TABLE temp_$TABLE;"
+
+
+done

devops/scripts/pg_scripts/db_to_csv.sh

@@ -0,0 +1,152 @@
+
+do_dash() {
+   echo "---------------------------------------------------------------"
+}
+
+SAVE=true
+
+# 983,523,5780,286,1088,5014,4626,2854,1125,1858,2169,4443,3035,4179,3866,1774,1308,5667,3156,5498,4992,5831,4828,5639,3566,4811,3518,2014,3271,2871,5655,5447,4324,2338,2283,1770,3736,3605,1702,704,2843,1179,5580,2679,4083,5601,2051,2893,1999,546,4226,3109,6035,4465,4090,1707,1019,2157,4983,2156,886,4153,553,2543,1970,2759,2632,5839,4539,4531,4746,1783,3323,2199,5958,830,2638,2072,5417,1367,1192,5059,2890,1877,1782,5651,4743,4111,897,339,5583,3562,3448,3098,188,5523,5582,4784,2770,6127
+do_dash; echo "\n Defining variables"
+
+RDS_LIST=( "prod-05-jun" "prod-06-jun" "prod-07-jun" "prod-08-jun" "prod-09-jun"
+           "prod-10-jun" "prod-11-jun" "prod-snapshot-before-12-jun-deploy" "prod-12-jun" )
+PROJECT_IDS=( "(4995)" "(4413,5365,5401,5402,5474,5501,5604,5605,5740,5773,5795,5840,5890,5999,6000,6002)"
+             "(1945,3744,5127,5375,5685,5762,5956,5968,5969,5978,6067)"
+             "(3901,5808,5844)" "(5953,5692,5169,4166,3904)"
+             "(6015,6005,5948,5274,4411,4392,4259)"
+             "(2742,3389,5340,5455,5652)" "(5942,4351)"
+             "(2469,4597,5509,5660,5831,5895,5907,5964,6059,6061)" )
+IDX=0
+
+# Create folder.
+echo "Creating folders"
+FOLDER_PATH=dump/lost_projects
+mkdir -p $FOLDER_PATH
+
+
+for HOST in "${RDS_LIST[@]}"; do
+  do_dash; echo "Priority areas tables"
+  TABLE=priority_areas
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  echo $HOST
+  echo ${PROJECT_IDS[IDX]}
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE id in
+  		(SELECT priority_area_id from project_priority_areas where project_id in ${PROJECT_IDS[IDX]}))
+  		to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Projects table"
+  TABLE=projects
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project allowed users"
+  TABLE=project_allowed_users
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Tasks table"
+  TABLE=tasks
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Task history"
+  TABLE=task_history
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Task invalidation history"
+  TABLE=task_invalidation_history
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project priority areas"
+  TABLE=project_priority_areas
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project chat"
+  TABLE=project_chat
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Project info"
+  TABLE=project_info
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ###############################################################################################
+
+  do_dash; echo "Messages"
+  TABLE=messages
+  mkdir -p $FOLDER_PATH/$HOST
+  FILE=$FOLDER_PATH/$HOST/$TABLE.csv
+  psql "host=$HOST.$ENDPOINT dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+  	"\copy (SELECT * FROM $TABLE WHERE project_id in ${PROJECT_IDS[IDX]}) to '$FILE' DELIMITER ',' CSV HEADER"
+
+  ((IDX++))
+done
+
+
+
+###############################################################################################
+# TABLE=users
+# do_dash; echo "users from tables"
+# psql "host=$HOST dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+# 	"copy(SELECT * FROM $TABLE WHERE id in
+# 		(select author_id from projects where id in $PROJECT_IDS)
+# 		OR id in (select users.id from users,project_allowed_users where users.id=project_allowed_users.user_id AND project_allowed_users.project_id in $PROJECT_IDS)
+# 		OR id in (select users.id from users,task_history where users.id=task_history.user_id AND task_history.project_id in $PROJECT_IDS)
+# 		OR id in (select users.id from users,tasks where users.id in (tasks.locked_by, tasks.validated_by, tasks.mapped_by) AND tasks.project_id in $PROJECT_IDS))
+# 		to '$FILE' DELIMITER ',' CSV HEADER"
+
+# if $SAVE; then
+# 	save_to_db
+# fi
+
+
+# ###############################################################################################
+
+# do_dash; echo "licenses from tables"
+# TABLE=licenses
+# psql "host=$HOST dbname=$DB user=$DB_USER password=$DB_PASSWORD" -c \
+# 	"copy(SELECT * FROM $TABLE WHERE id in (select license_id from projects where id in $PROJECT_IDS)) to '$FILE' DELIMITER ',' CSV HEADER"
+
+# if $SAVE; then
+# 	save_to_db
+# fi
+
+###############################################################################################

devops/scripts/pg_scripts/restart_table.sh

@@ -0,0 +1,6 @@
+NEW_DB='tm_new'
+
+echo $NEW_DB
+psql -U $DB_USER -d $DB -c "DROP DATABASE $NEW_DB"
+psql -U $DB_USER -d $DB -c "CREATE DATABASE $NEW_DB"
+psql -U $DB_USER -d $NEW_DB -c "CREATE EXTENSION postgis;"

manage.py

@@ -10,22 +10,22 @@
 
 import os
 import warnings
- 
+
 # Load configuration from file
 load_dotenv(os.path.join(os.path.dirname(__file__), 'tasking-manager.env'))
 
 # Temporarily here - to support backwards compatibility with TM_DB key.
 if os.getenv('TM_DB', False):
-    for key in ['TM_APP_BASE_URL','TM_SECRET','TM_CONSUMER_KEY','TM_CONSUMER_SECRET']:
+    for key in ['TM_APP_BASE_URL', 'TM_SECRET', 'TM_CONSUMER_KEY', 'TM_CONSUMER_SECRET']:
         if not os.getenv(key):
             warnings.warn("%s environmental variable not set." % (key,))
 else:
     # Check that required environmental variables are set
-    for key in ['TM_APP_BASE_URL','POSTGRES_DB','POSTGRES_USER','POSTGRES_PASSWORD','TM_SECRET','TM_CONSUMER_KEY','TM_CONSUMER_SECRET']:
+    for key in ['TM_APP_BASE_URL', 'POSTGRES_DB', 'POSTGRES_USER', 'POSTGRES_PASSWORD', 'TM_SECRET',
+                'TM_CONSUMER_KEY', 'TM_CONSUMER_SECRET', 'TM_DEFAULT_CHANGESET_COMMENT']:
         if not os.getenv(key):
             warnings.warn("%s environmental variable not set." % (key,))
 
-
 # Initialise the flask app object
 application = create_app()
 try:

migrations/versions/b25f088d40c2_.py

@@ -17,6 +17,10 @@
 
 
 def upgrade():
+    # Remove all task ids in messages for tasks that don't exists anynmore
+    query = 'UPDATE messages SET task_id = NULL WHERE task_id NOT IN (SELECT id FROM tasks WHERE project_id = messages.project_id);'
+    op.execute(query)
+
     op.create_foreign_key('messages_tasks', 'messages', 'tasks', ['task_id', 'project_id'], ['id', 'project_id'])
 
 

server/config.py

@@ -2,6 +2,7 @@
 import os
 from dotenv import load_dotenv
 
+
 class EnvironmentConfig:
     """ Base class for configuration. """
     """ Most settings can be defined through environment variables. """
@@ -13,7 +14,7 @@ class EnvironmentConfig:
     APP_BASE_URL = os.getenv('TM_APP_BASE_URL', 'http://127.0.0.1:5000')
 
     # The default tag used in the OSM changeset comment
-    DEFAULT_CHANGESET_COMMENT = os.getenv('TM_DEFAULT_CHANGESET_COMMENT', '#tm-project')
+    DEFAULT_CHANGESET_COMMENT = os.getenv('TM_DEFAULT_CHANGESET_COMMENT', None)
 
     # The address to use as the sender on auto generated emails
     EMAIL_FROM_ADDRESS = os.getenv('TM_EMAIL_FROM_ADDRESS', None)