From 337c87cd916de8f2538dd06a047ee356b1008fdc Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Fri, 2 Oct 2020 12:58:18 +0200 Subject: [PATCH] F #4985: Support for cluster replicas to ssh driver The initial caching mechanism will be exteneded with: - Cache remplacement policy - Optimize recovery snapshot with qcow2 format - Add support for automatic snapshot VM disks co-authored-by: Jan Orel --- install.sh | 3 + share/etc/oned.conf | 2 +- src/datastore_mad/remotes/fs/rm | 4 +- src/datastore_mad/remotes/fs/snap_delete | 2 + src/datastore_mad/remotes/fs/snap_flatten | 2 + src/datastore_mad/remotes/fs/snap_revert | 2 + src/tm_mad/ssh/clone | 17 ++- src/tm_mad/ssh/clone.replica | 161 ++++++++++++++++++++++ src/tm_mad/ssh/ln.replica | 1 + src/tm_mad/ssh/mvds | 2 + src/tm_mad/ssh/snap_create | 61 ++++---- src/tm_mad/ssh/snap_create_live | 88 +++++++++++- src/tm_mad/ssh/sshrc | 40 ++++++ src/tm_mad/tm_common.sh | 109 ++++++++++++++- src/vmm_mad/exec/vmm_execrc | 2 +- 15 files changed, 454 insertions(+), 42 deletions(-) create mode 100755 src/tm_mad/ssh/clone.replica create mode 120000 src/tm_mad/ssh/ln.replica mode change 120000 => 100755 src/tm_mad/ssh/snap_create_live create mode 100644 src/tm_mad/ssh/sshrc diff --git a/install.sh b/install.sh index 25bf7d5f978..78319c2571e 100755 --- a/install.sh +++ b/install.sh @@ -1651,8 +1651,11 @@ TM_QCOW2_FILES="src/tm_mad/qcow2/clone \ src/tm_mad/qcow2/resize" TM_SSH_FILES="src/tm_mad/ssh/clone \ + src/tm_mad/ssh/clone.replica \ + src/tm_mad/ssh/sshrc \ src/tm_mad/ssh/delete \ src/tm_mad/ssh/ln \ + src/tm_mad/ssh/ln.replica \ src/tm_mad/ssh/mkswap \ src/tm_mad/ssh/mkimage \ src/tm_mad/ssh/mv \ diff --git a/share/etc/oned.conf b/share/etc/oned.conf index be941640cf0..8a52610c317 100644 --- a/share/etc/oned.conf +++ b/share/etc/oned.conf @@ -1154,7 +1154,7 @@ TM_MAD_CONF = [ TM_MAD_CONF = [ NAME = "ssh", LN_TARGET = "SYSTEM", CLONE_TARGET = "SYSTEM", SHARED = "NO", - DS_MIGRATE = "YES" + DS_MIGRATE = "YES", ALLOW_ORPHANS="YES" ] TM_MAD_CONF = [ diff --git a/src/datastore_mad/remotes/fs/rm b/src/datastore_mad/remotes/fs/rm index 2ef582e52b6..885ef6dec20 100755 --- a/src/datastore_mad/remotes/fs/rm +++ b/src/datastore_mad/remotes/fs/rm @@ -59,7 +59,7 @@ BASE_PATH="${XPATH_ELEMENTS[i++]}" if [ -n "$BRIDGE_LIST" ]; then DST_HOST=`get_destination_host $ID` - ssh_exec_and_log "$DST_HOST" "[ -f $SRC ] && rm -rf $SRC $SRC.snap" \ + ssh_exec_and_log "$DST_HOST" "[ -f $SRC ] && rm -rf $SRC $SRC.snap $SRC.md5sum" \ "Error deleting $SRC in $DST_HOST" else BASENAME_SRC=`basename "${SRC##$REMOTE_RM_CMD}"` @@ -67,7 +67,7 @@ else then log "Removing $SRC from the image repository" - exec_and_log "rm -rf $SRC $SRC.snap" \ + exec_and_log "rm -rf $SRC $SRC.snap $SRC.md5sum" \ "Error deleting $SRC" else log_error "Bad formed or unavailable Image source: ${SRC}" diff --git a/src/datastore_mad/remotes/fs/snap_delete b/src/datastore_mad/remotes/fs/snap_delete index 66ff82391b0..cd7e5aea3ee 100755 --- a/src/datastore_mad/remotes/fs/snap_delete +++ b/src/datastore_mad/remotes/fs/snap_delete @@ -71,4 +71,6 @@ else exec_and_log "rm ${SNAP_PATH}" \ "Error deleting snapshot $SNAP_PATH" + + rm -f ${SRC_PATH}.md5sum fi diff --git a/src/datastore_mad/remotes/fs/snap_flatten b/src/datastore_mad/remotes/fs/snap_flatten index 6eefa6a41ee..ddf9adfd358 100755 --- a/src/datastore_mad/remotes/fs/snap_flatten +++ b/src/datastore_mad/remotes/fs/snap_flatten @@ -103,5 +103,7 @@ else exec_and_log "rm -rf ${SNAP_DIR}" \ "Error removing ${SNAP_DIR}" + + rm -f ${SRC_PATH}.md5sum fi fi diff --git a/src/datastore_mad/remotes/fs/snap_revert b/src/datastore_mad/remotes/fs/snap_revert index 849a3951e64..fa333b27ca4 100755 --- a/src/datastore_mad/remotes/fs/snap_revert +++ b/src/datastore_mad/remotes/fs/snap_revert @@ -104,5 +104,7 @@ else exec_and_log "mv ${DISK_SRC}.tmp ${DISK_SRC}" \ "Error moving to $DISK_SRC" + + rm -f ${DISK_SRC}.md5sum fi fi diff --git a/src/tm_mad/ssh/clone b/src/tm_mad/ssh/clone index dd89c76ce09..de11ba49deb 100755 --- a/src/tm_mad/ssh/clone +++ b/src/tm_mad/ssh/clone @@ -39,6 +39,7 @@ fi . $TMCOMMON DRIVER_PATH=$(dirname $0) +XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" #------------------------------------------------------------------------------- # Set dst path and dir @@ -63,18 +64,28 @@ ssh_make_path $DST_HOST $DST_DIR "ssh" DISK_ID=$(basename ${DST_PATH} | cut -d. -f2) -XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" - unset i j XPATH_ELEMENTS while IFS= read -r -d '' element; do XPATH_ELEMENTS[i++]="$element" done < <(onevm show -x $VMID| $XPATH \ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE \ - /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE) + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE \ + /VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID) SIZE="${XPATH_ELEMENTS[j++]}" ORIGINAL_SIZE="${XPATH_ELEMENTS[j++]}" +SYS_DS_ID="${XPATH_ELEMENTS[j++]}" + +#------------------------------------------------------------------------------- +# Check for REPLICA_HOST in DATASTORE TEMPLATE and exec ./$0.replica if found +#------------------------------------------------------------------------------- +REPLICA_HOST=$(get_replica_host $SYS_DS_ID) + +if [ -n "$REPLICA_HOST" ]; then + $0.replica $@ "$REPLICA_HOST" + exit 0 +fi #------------------------------------------------------------------------------- # Copy files to the remote host diff --git a/src/tm_mad/ssh/clone.replica b/src/tm_mad/ssh/clone.replica new file mode 100755 index 00000000000..8fe52597da9 --- /dev/null +++ b/src/tm_mad/ssh/clone.replica @@ -0,0 +1,161 @@ +#!/bin/bash + +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# clone fe:SOURCE host:remote_system_ds/disk.i vmid dsid [replica_host] +# - fe is the front-end hostname +# - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk +# - host is the target host to deploy the VM +# - remote_system_ds is the path for the system datastore in the host +# - vmid is the id of the VM +# - dsid is the target datastore (0 is the system datastore +# - replica_host is an optional paramter when called from the tm/ssh/clone + +SRC=$1 +DST=$2 + +VMID=$3 +DSID=$4 + +REPLICA_HOST=$5 + +if [ -z "${ONE_LOCATION}" ]; then + TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh + SSH_RC=/var/lib/one/remotes/tm/ssh/sshrc +else + TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh + SSH_RC=$ONE_LOCATION/var/remotes/tm/ssh/sshrc +fi + +. $TMCOMMON +. $SSH_RC + +DRIVER_PATH=$(dirname $0) +XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" + +#------------------------------------------------------------------------------- +# Set dst path and dir +#------------------------------------------------------------------------------- +SRC_PATH=$(arg_path $SRC) +DST_PATH=$(arg_path $DST) +SRC_FILE=$(basename $SRC_PATH) +DST_FILE=$(basename $DST_PATH) + +SRC_HOST=$(arg_host $SRC) +DST_HOST=$(arg_host $DST) + +SRC_DIR=$(dirname $SRC_PATH) +DST_DIR=$(dirname $DST_PATH) + +#------------------------------------------------------------------------------- +# Get Image and System DS information +#------------------------------------------------------------------------------- +DISK_ID=$(basename ${DST_PATH} | cut -d. -f2) + +unset i j XPATH_ELEMENTS + +while IFS= read -r -d '' element; do + XPATH_ELEMENTS[i++]="$element" +done < <(onevm show -x $VMID| $XPATH \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE \ + /VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID) + +SIZE="${XPATH_ELEMENTS[j++]}" +ORIGINAL_SIZE="${XPATH_ELEMENTS[j++]}" +SYS_DS_ID="${XPATH_ELEMENTS[j++]}" + +# ------------------------------------------------------------------------------ +# Get REPLICA_HOST from DATASTORE (if not passed as 5th arg) +# ------------------------------------------------------------------------------ +if [ -z "$REPLICA_HOST" ]; then + REPLICA_HOST=$(get_replica_host $SYS_DS_ID) + + if [ -z "$REPLICA_HOST" ]; then + error_message "No REPLICA_HOST in datastore $SYS_DS_ID template" + exit 1 + fi +fi + +#------------------------------------------------------------------------------- +# Create DST path +#------------------------------------------------------------------------------- +ssh_make_path $DST_HOST $DST_DIR "ssh" + +# ------------------------------------------------------------------------------ +# Get REPLICA_STORAGE_IP from host template +# ------------------------------------------------------------------------------ +REPLICA_STORAGE_IP=$(awk 'gsub(/[\0]/, x)' \ + <( onehost show $DST_HOST -x | $XPATH /HOST/TEMPLATE/REPLICA_STORAGE_IP )) + +# ------------------------------------------------------------------------------ +# Synchronize Image Datastore in the Replica Host. Use a recovery snapshot +# if present in the RECOVERY_SNAPS_DIR +# ------------------------------------------------------------------------------ +if ssh $REPLICA_HOST \ + "test -e ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID/${DST_FILE}.replica_snap"; then + + # point to [disk].replica_snap files + SRC_DIR=${REPLICA_RECOVERY_SNAPS_DIR}/$VMID + SRC_FILE="${DST_FILE}.replica_snap" +else + rsync_img_to_replica "$SRC_PATH" "$REPLICA_HOST" + + if [ -n "$ORIGINAL_SIZE" -a "$SIZE" -gt "$ORIGINAL_SIZE" ]; then + RESIZE_CMD="qemu-img resize ${DST_PATH} ${SIZE}M" + fi +fi + +#------------------------------------------------------------------------------- +# Clone (tar|ssh) SRC into DST +#------------------------------------------------------------------------------- +log "Cloning $SRC_PATH via replica $REPLICA_HOST in $DST" + +# copy locally, we hit the replica +if [ "$REPLICA_HOST" = "$DST_HOST" ]; then + CLONE_CMD=$(cat </dev/null 2>&1; then exec_and_log "rsync -r --delete ${SRC_HOST}:${SRC_PATH_SNAP}/ ${DST_SNAP}" fi +rm -f ${DST}.md5sum + exit 0 diff --git a/src/tm_mad/ssh/snap_create b/src/tm_mad/ssh/snap_create index e83d8123c83..031a85aedc2 100755 --- a/src/tm_mad/ssh/snap_create +++ b/src/tm_mad/ssh/snap_create @@ -21,64 +21,59 @@ SRC=$1 SNAP_ID=$2 VMID=$3 -DSID=$4 if [ -z "${ONE_LOCATION}" ]; then TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh - DATASTORES=/var/lib/one/datastores + SSH_RC=/var/lib/one/remotes/tm/ssh/sshrc else TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh - DATASTORES=$ONE_LOCATION/var/datastores + SSH_RC=$ONE_LOCATION/var/remotes/tm/ssh/sshrc fi DRIVER_PATH=$(dirname $0) -. $TMCOMMON +source $TMCOMMON +source $SSH_RC -SRC_PATH=$(arg_path $SRC) +SRC_PATH=$(arg_path $SRC NOFIX) SRC_HOST=$(arg_host $SRC) #------------------------------------------------------------------------------- -# Get Image information +# Get VM information #------------------------------------------------------------------------------- -DISK_ID=$(basename ${SRC} | cut -d. -f2) - XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" -unset i j XPATH_ELEMENTS - -while IFS= read -r -d '' element; do - XPATH_ELEMENTS[i++]="$element" -done < <(onevm show -x $VMID| $XPATH \ - /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \ - /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE) - -DISK_SRC="${XPATH_ELEMENTS[j++]}" -CLONE="${XPATH_ELEMENTS[j++]}" - - -SYSTEM_DS_PATH=$(dirname ${SRC_PATH}) -IMAGE_DS_PATH=$(dirname ${DISK_SRC}) - -DISK_PATH="${SYSTEM_DS_PATH}/disk.${DISK_ID}" +DS_ID=$(awk 'gsub(/[\0]/, x)' \ + <(onevm show $VMID -x|$XPATH /VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID)) -SNAP_DIR="${DISK_PATH}.snap" +VM_DIR="$(dirname ${SRC_PATH})" +DISK_NAME="$(basename $SRC_PATH)" +SNAP_DIR="${SRC_PATH}.snap" SNAP_PATH="${SNAP_DIR}/${SNAP_ID}" -SNAP_PATH_RELATIVE=$(basename ${SNAP_PATH}) -CURRENT_PATH=${DISK_PATH} + +REPLICA_HOST=$(get_replica_host $DS_ID) CMD=$(cat < "${IMG_PATH}.md5sum" + fi + + SRC_MD5SUM=$(cat "${IMG_PATH}.md5sum") + + # replica can not reuse md5sum_with_snaps, duplicate it + SCRIPT=$(cat </dev/null | cut -d " " -f 1 > ${IMG_PATH}.md5sum + else + CMD="md5sum $IMG_PATH" + + for S in \$SNAPS; do + CMD+=" ${IMG_PATH}.snap/${S}" + done + + \$CMD | md5sum | cut -d " " -f 1 > ${IMG_PATH}.md5sum + fi + + cat ${IMG_PATH}.md5sum + fi +EOF +) + + DST_MD5SUM=$(ssh "$REPLICA_HOST" "$SCRIPT") + + [ "$SRC_MD5SUM" != "$DST_MD5SUM" ] +} + +# ------------------------------------------------------------------------------ +# Rsync (tar|ssh) IMG_PATH to the REPLICA_HOST +# ------------------------------------------------------------------------------ +function rsync_img_to_replica { + local IMG_PATH="$1" + local REPLICA_HOST="$2" + local LOCK_TIMEOUT="${REPLICA_COPY_LOCK_TIMEOUT:-600}" + local DST_DIR=$(dirname $IMG_PATH) + + ssh_make_path $REPLICA_HOST $DST_DIR + + # sync to replica, include .md5sum and .snap dir + LOCK="$REPLICA_HOST-${IMG_PATH//\//-}" + if exclusive "$LOCK" "$LOCK_TIMEOUT" repl_img_outdated $IMG_PATH $REPLICA_HOST; then + LOCK="replica-$REPLICA_HOST-${IMG_PATH//\//-}" + RSYNC_CMD=$(cat <