Update Cassandra backup/restore scripts (#3)

wireapp · Apr 3, 2019 · 69ad537 · 69ad537
1 parent 5d17887
commit 69ad537
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 11 deletions.
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -60,7 +60,7 @@ cassandra_seed_groupname: cassandra_seed
 cassandra_groupname: cassandra
 cassandra_seeds: []
 cassandra_seed_resolved: '{% if cassandra_seed_resolution == "static" %}{{ cassandra_seeds | join(",")}}{% else %}{% for host in groups[cassandra_seed_groupname] %}{{hostvars[host]["ansible_" + cassandra_network_interface]["ipv4"]["address"]}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}'
-cassandra_all_instances: '{{ groups[cassandra_seed_groupname]|union(groups[cassandra_groupname])|reverse|list }}'
+cassandra_all_instances: '{{ groups[cassandra_seed_groupname]|union(groups[cassandra_groupname]|default([]))|reverse|list }}'
 
 # start cassandra immediately via systemd?
 cassandra_activate: true

diff --git a/templates/cassandra_backup.j2 b/templates/cassandra_backup.j2
@@ -23,8 +23,6 @@ backup_base="{{ cassandra_env }}-{{ cassandra_role }}/${day_bucket}/${host}/${ti
 empty_file=/tmp/empty
 done_msg=BACKUP_DONE_SUCCESSFULLY
 
-par=2
-
 metrics_dir=/var/lib/node-exporter/textfile-collector
 mkdir -p -m 777 $metrics_dir
 metrics_file=${metrics_dir}/cassandra_backup.prom
@@ -78,12 +76,12 @@ function retry3 { $1 || $1 || $1; }
 export -f retry3
 
 # Upload to S3
-for local_file in $base/*/*/$snaps/$sid/*; do
-       snapshot_subdir_removed=${local_file//$snaps\/$sid\/};
+for snapshot_subdir in $base/*/*/$snaps/$sid/; do
+       snapshot_subdir_removed=${snapshot_subdir//$snaps\/$sid\/};
        key_name=$backup_base${snapshot_subdir_removed#$base}
-       echo $bucket $key_name $local_file;
-done | xargs -P$par -n3 bash -c \
-  'retry3 "aws s3 cp $3 s3://$1/$2 --expected-size $(stat --printf='%s' $3) --no-guess-mime-type" || (echo Upload failed! && exit 255)' -
+       echo $bucket $key_name $snapshot_subdir;
+done | xargs -n3 bash -c \
+  'retry3 "aws s3 sync $3 s3://$1/$2 --no-guess-mime-type" || (echo Upload failed! && exit 255)' -
 
 #################################
 #### Mark remote as done  #######

diff --git a/templates/cassandra_restore.j2 b/templates/cassandra_restore.j2
@@ -19,9 +19,13 @@ if [ -z ${SOURCE_HOSTNAME+x} ]; then echo "Set SOURCE_HOSTNAME to the node's hos
 if [ -z ${SOURCE_BACKUP_DATE+x} ]; then echo "Set SOURCE_BACKUP_DATE to the backup date, in format YYYY-DD-MM"; exit 1; fi
 if [ -z ${SOURCE_BACKUP_TIME+x} ]; then echo "Set SOURCE_BACKUP_TIME to the node's backup time, in format HH:MM:SS"; exit 1; fi
 
-S3_BACKUP_PATH=/$SOURCE_ENV-$SOURCE_ROLE/$SOURCE_BACKUP_DATE/$SOURCE_HOSTNAME/$SOURCE_BACKUP_TIME
+S3_BACKUP_PATH=$SOURCE_ENV-$SOURCE_ROLE/$SOURCE_BACKUP_DATE/$SOURCE_HOSTNAME/$SOURCE_BACKUP_TIME
 
-aws s3 sync "s3://{{ cassandra_backup_s3_bucket }}/$S3_BACKUP_PATH/$CASSANDRA_KSP/$CASSANDRA_CF" "$TMP_WORKING_DIR"
+mkdir -p $TMP_WORKING_DIR/$CASSANDRA_KSP/
 
-"$SSTABLE_LOADER" -d "$CASSANDRA_HOST" -f "$CASSANDRA_YAML" "$TMP_WORKING_DIR/$S3_BACKUP_PATH/$CASSANDRA_KSP/$CASSANDRA_CF/"
+aws s3 sync "s3://{{ cassandra_backup_s3_bucket }}/$S3_BACKUP_PATH/$CASSANDRA_KSP/" "$TMP_WORKING_DIR/$CASSANDRA_KSP/"
 
+# Note: Folder names for cassandra 3.11 are composed of <column family>-<some hash>, the hash is unknown in advance (or is it?)
+# Use `-*` bash extension to guess the hash part.
+# Relies on the column family name not having a "-" in it
+"$SSTABLE_LOADER" -d "$CASSANDRA_HOST" -f "$CASSANDRA_YAML" $TMP_WORKING_DIR/$CASSANDRA_KSP/$CASSANDRA_CF-*/