F OpenNebula#4985: Support for cluster replicas to ssh driver

The initial caching mechanism will be exteneded with: - Cache remplacement policy - Optimize recovery snapshot with qcow2 format - Add support for automatic snapshot VM disks co-authored-by: Jan Orel <[email protected]>
storpool · Oct 16, 2020 · 4062ff5 · 4062ff5
1 parent a8adbe2
commit 4062ff5
Show file tree

Hide file tree

Showing 15 changed files with 454 additions and 42 deletions.
diff --git a/install.sh b/install.sh
@@ -1651,8 +1651,11 @@ TM_QCOW2_FILES="src/tm_mad/qcow2/clone \
                  src/tm_mad/qcow2/resize"
 
 TM_SSH_FILES="src/tm_mad/ssh/clone \
+              src/tm_mad/ssh/clone.replica \
+              src/tm_mad/ssh/sshrc \
               src/tm_mad/ssh/delete \
               src/tm_mad/ssh/ln \
+              src/tm_mad/ssh/ln.replica \
               src/tm_mad/ssh/mkswap \
               src/tm_mad/ssh/mkimage \
               src/tm_mad/ssh/mv \

diff --git a/share/etc/oned.conf b/share/etc/oned.conf
@@ -1154,7 +1154,7 @@ TM_MAD_CONF = [
 
 TM_MAD_CONF = [
     NAME = "ssh", LN_TARGET = "SYSTEM", CLONE_TARGET = "SYSTEM", SHARED = "NO",
-    DS_MIGRATE = "YES"
+    DS_MIGRATE = "YES", ALLOW_ORPHANS="YES"
 ]
 
 TM_MAD_CONF = [

diff --git a/src/datastore_mad/remotes/fs/rm b/src/datastore_mad/remotes/fs/rm
@@ -59,15 +59,15 @@ BASE_PATH="${XPATH_ELEMENTS[i++]}"
 if [ -n "$BRIDGE_LIST" ]; then
     DST_HOST=`get_destination_host $ID`
 
-    ssh_exec_and_log "$DST_HOST" "[ -f $SRC ] && rm -rf $SRC $SRC.snap" \
+    ssh_exec_and_log "$DST_HOST" "[ -f $SRC ] && rm -rf $SRC $SRC.snap $SRC.md5sum" \
         "Error deleting $SRC in $DST_HOST"
 else
     BASENAME_SRC=`basename "${SRC##$REMOTE_RM_CMD}"`
     if [ -f "$SRC" -a `dirname "$SRC"` = "$BASE_PATH" -a -n "$BASENAME_SRC" ]
     then
         log "Removing $SRC from the image repository"
 
-        exec_and_log "rm -rf $SRC $SRC.snap" \
+        exec_and_log "rm -rf $SRC $SRC.snap $SRC.md5sum" \
             "Error deleting $SRC"
     else
         log_error "Bad formed or unavailable Image source: ${SRC}"

diff --git a/src/datastore_mad/remotes/fs/snap_delete b/src/datastore_mad/remotes/fs/snap_delete
@@ -71,4 +71,6 @@ else
 
     exec_and_log "rm ${SNAP_PATH}" \
         "Error deleting snapshot $SNAP_PATH"
+
+    rm -f ${SRC_PATH}.md5sum
 fi
diff --git a/src/datastore_mad/remotes/fs/snap_flatten b/src/datastore_mad/remotes/fs/snap_flatten
@@ -103,5 +103,7 @@ else
 
         exec_and_log "rm -rf ${SNAP_DIR}" \
             "Error removing ${SNAP_DIR}"
+
+        rm -f ${SRC_PATH}.md5sum
     fi
 fi
diff --git a/src/datastore_mad/remotes/fs/snap_revert b/src/datastore_mad/remotes/fs/snap_revert
@@ -104,5 +104,7 @@ else
 
         exec_and_log "mv ${DISK_SRC}.tmp ${DISK_SRC}" \
             "Error moving to $DISK_SRC"
+
+        rm -f ${DISK_SRC}.md5sum
     fi
 fi
diff --git a/src/tm_mad/ssh/clone b/src/tm_mad/ssh/clone
@@ -39,6 +39,7 @@ fi
 . $TMCOMMON
 
 DRIVER_PATH=$(dirname $0)
+XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
 
 #-------------------------------------------------------------------------------
 # Set dst path and dir
@@ -63,18 +64,28 @@ ssh_make_path $DST_HOST $DST_DIR "ssh"
 
 DISK_ID=$(basename ${DST_PATH} | cut -d. -f2)
 
-XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
-
 unset i j XPATH_ELEMENTS
 
 while IFS= read -r -d '' element; do
     XPATH_ELEMENTS[i++]="$element"
 done < <(onevm show -x $VMID| $XPATH \
                     /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE \
-                    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE)
+                    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE \
+                    /VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID)
 
 SIZE="${XPATH_ELEMENTS[j++]}"
 ORIGINAL_SIZE="${XPATH_ELEMENTS[j++]}"
+SYS_DS_ID="${XPATH_ELEMENTS[j++]}"
+
+#-------------------------------------------------------------------------------
+# Check for REPLICA_HOST in DATASTORE TEMPLATE and exec ./$0.replica if found
+#-------------------------------------------------------------------------------
+REPLICA_HOST=$(get_replica_host $SYS_DS_ID)
+
+if [ -n "$REPLICA_HOST" ]; then
+    $0.replica $@ "$REPLICA_HOST"
+    exit 0
+fi
 
 #-------------------------------------------------------------------------------
 # Copy files to the remote host

diff --git a/src/tm_mad/ssh/clone.replica b/src/tm_mad/ssh/clone.replica
@@ -0,0 +1,161 @@
+#!/bin/bash
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+# clone fe:SOURCE host:remote_system_ds/disk.i vmid dsid [replica_host]
+#   - fe is the front-end hostname
+#   - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk
+#   - host is the target host to deploy the VM
+#   - remote_system_ds is the path for the system datastore in the host
+#   - vmid is the id of the VM
+#   - dsid is the target datastore (0 is the system datastore
+#   - replica_host is an optional paramter when called from the tm/ssh/clone
+
+SRC=$1
+DST=$2
+
+VMID=$3
+DSID=$4
+
+REPLICA_HOST=$5
+
+if [ -z "${ONE_LOCATION}" ]; then
+    TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
+    SSH_RC=/var/lib/one/remotes/tm/ssh/sshrc
+else
+    TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
+    SSH_RC=$ONE_LOCATION/var/remotes/tm/ssh/sshrc
+fi
+
+. $TMCOMMON
+. $SSH_RC
+
+DRIVER_PATH=$(dirname $0)
+XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
+
+#-------------------------------------------------------------------------------
+# Set dst path and dir
+#-------------------------------------------------------------------------------
+SRC_PATH=$(arg_path $SRC)
+DST_PATH=$(arg_path $DST)
+SRC_FILE=$(basename $SRC_PATH)
+DST_FILE=$(basename $DST_PATH)
+
+SRC_HOST=$(arg_host $SRC)
+DST_HOST=$(arg_host $DST)
+
+SRC_DIR=$(dirname $SRC_PATH)
+DST_DIR=$(dirname $DST_PATH)
+
+#-------------------------------------------------------------------------------
+# Get Image and System DS information
+#-------------------------------------------------------------------------------
+DISK_ID=$(basename ${DST_PATH} | cut -d. -f2)
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <(onevm show -x $VMID| $XPATH \
+                    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE \
+                    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE \
+                    /VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID)
+
+SIZE="${XPATH_ELEMENTS[j++]}"
+ORIGINAL_SIZE="${XPATH_ELEMENTS[j++]}"
+SYS_DS_ID="${XPATH_ELEMENTS[j++]}"
+
+# ------------------------------------------------------------------------------
+# Get REPLICA_HOST from DATASTORE (if not passed as 5th arg)
+# ------------------------------------------------------------------------------
+if [ -z "$REPLICA_HOST" ]; then
+    REPLICA_HOST=$(get_replica_host $SYS_DS_ID)
+
+    if [ -z "$REPLICA_HOST" ]; then
+        error_message "No REPLICA_HOST in datastore $SYS_DS_ID template"
+        exit 1
+    fi
+fi
+
+#-------------------------------------------------------------------------------
+# Create DST path
+#-------------------------------------------------------------------------------
+ssh_make_path $DST_HOST $DST_DIR "ssh"
+
+# ------------------------------------------------------------------------------
+# Get REPLICA_STORAGE_IP from host template
+# ------------------------------------------------------------------------------
+REPLICA_STORAGE_IP=$(awk 'gsub(/[\0]/, x)' \
+    <( onehost show $DST_HOST -x | $XPATH /HOST/TEMPLATE/REPLICA_STORAGE_IP ))
+
+# ------------------------------------------------------------------------------
+#  Synchronize Image Datastore in the Replica Host. Use a recovery snapshot
+#  if present in the RECOVERY_SNAPS_DIR
+# ------------------------------------------------------------------------------
+if ssh $REPLICA_HOST \
+    "test -e ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID/${DST_FILE}.replica_snap"; then
+
+    # point to [disk].replica_snap files
+    SRC_DIR=${REPLICA_RECOVERY_SNAPS_DIR}/$VMID
+    SRC_FILE="${DST_FILE}.replica_snap"
+else
+    rsync_img_to_replica "$SRC_PATH" "$REPLICA_HOST"
+
+    if [ -n "$ORIGINAL_SIZE" -a "$SIZE" -gt "$ORIGINAL_SIZE" ]; then
+       RESIZE_CMD="qemu-img resize ${DST_PATH} ${SIZE}M"
+    fi
+fi
+
+#-------------------------------------------------------------------------------
+# Clone (tar|ssh) SRC into DST
+#-------------------------------------------------------------------------------
+log "Cloning $SRC_PATH via replica $REPLICA_HOST in $DST"
+
+# copy locally, we hit the replica
+if [ "$REPLICA_HOST" = "$DST_HOST" ]; then
+    CLONE_CMD=$(cat <<EOF
+        cd $SRC_DIR
+        cp $SRC_FILE $DST_PATH
+
+        if [ -d $SRC_PATH.snap ]; then
+            cp -r $SRC_PATH.snap $DST_PATH.snap
+        fi
+EOF
+)
+else
+    # copy to remote using faster tar|ssh
+    CLONE_CMD=$(cat <<EOF
+        if [ -d "${SRC_PATH}.snap" ]; then
+            SRC_SNAP="${SRC_FILE}.snap"
+        fi
+
+        cd $SRC_DIR
+        tar --transform="flags=r;s|$SRC_FILE|$DST_FILE|" -cSf - $SRC_FILE \$SRC_SNAP | \
+            ssh $REPLICA_SSH_OPTS ${REPLICA_STORAGE_IP:-$DST_HOST} "tar xSf - -C $DST_DIR"
+EOF
+)
+fi
+
+ssh_forward ssh_exec_and_log $REPLICA_HOST "$CLONE_CMD" \
+    "Error copying $SRC to $DST"
+
+if [ -n "$RESIZE_CMD" ]; then
+   ssh_exec_and_log "$DST_HOST" "$RESIZE_CMD" \
+       "Error resizing image $DST"
+fi
+
+exit 0
diff --git a/src/tm_mad/ssh/ln.replica b/src/tm_mad/ssh/ln.replica
@@ -0,0 +1 @@
+clone.replica
diff --git a/src/tm_mad/ssh/mvds b/src/tm_mad/ssh/mvds
@@ -55,4 +55,6 @@ if $SSH $SRC_HOST ls ${SRC_PATH_SNAP} >/dev/null 2>&1; then
     exec_and_log "rsync -r --delete ${SRC_HOST}:${SRC_PATH_SNAP}/ ${DST_SNAP}"
 fi
 
+rm -f ${DST}.md5sum
+
 exit 0
diff --git a/src/tm_mad/ssh/snap_create b/src/tm_mad/ssh/snap_create
@@ -21,64 +21,59 @@
 SRC=$1
 SNAP_ID=$2
 VMID=$3
-DSID=$4
 
 if [ -z "${ONE_LOCATION}" ]; then
     TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
-    DATASTORES=/var/lib/one/datastores
+    SSH_RC=/var/lib/one/remotes/tm/ssh/sshrc
 else
     TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
-    DATASTORES=$ONE_LOCATION/var/datastores
+    SSH_RC=$ONE_LOCATION/var/remotes/tm/ssh/sshrc
 fi
 
 DRIVER_PATH=$(dirname $0)
 
-. $TMCOMMON
+source $TMCOMMON
+source $SSH_RC
 
-SRC_PATH=$(arg_path $SRC)
+SRC_PATH=$(arg_path $SRC NOFIX)
 SRC_HOST=$(arg_host $SRC)
 
 #-------------------------------------------------------------------------------
-# Get Image information
+# Get VM information
 #-------------------------------------------------------------------------------
 
-DISK_ID=$(basename ${SRC} | cut -d. -f2)
-
 XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
 
-unset i j XPATH_ELEMENTS
-
-while IFS= read -r -d '' element; do
-    XPATH_ELEMENTS[i++]="$element"
-done < <(onevm show -x $VMID| $XPATH \
-                    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \
-                    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE)
-
-DISK_SRC="${XPATH_ELEMENTS[j++]}"
-CLONE="${XPATH_ELEMENTS[j++]}"
-
-
-SYSTEM_DS_PATH=$(dirname ${SRC_PATH})
-IMAGE_DS_PATH=$(dirname ${DISK_SRC})
-
-DISK_PATH="${SYSTEM_DS_PATH}/disk.${DISK_ID}"
+DS_ID=$(awk 'gsub(/[\0]/, x)' \
+    <(onevm show $VMID -x|$XPATH /VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID))
 
-SNAP_DIR="${DISK_PATH}.snap"
+VM_DIR="$(dirname ${SRC_PATH})"
+DISK_NAME="$(basename $SRC_PATH)"
+SNAP_DIR="${SRC_PATH}.snap"
 SNAP_PATH="${SNAP_DIR}/${SNAP_ID}"
-SNAP_PATH_RELATIVE=$(basename ${SNAP_PATH})
-CURRENT_PATH=${DISK_PATH}
+
+REPLICA_HOST=$(get_replica_host $DS_ID)
 
 CMD=$(cat <<EOT
 set -ex -o pipefail
 
-if [ ! -d "${SNAP_DIR}" ]; then
-    mkdir "${SNAP_DIR}"
-fi
+mkdir -p "${SNAP_DIR}"
+touch $SNAP_PATH
 
 # Copy current state to a new snapshot
-cp "${CURRENT_PATH}" "${SNAP_PATH}"
+cp "${SRC_PATH}" "${SNAP_PATH}"
+
+if [ -n "$REPLICA_HOST" ]; then
+    ssh $REPLICA_HOST "mkdir -p $REPLICA_RECOVERY_SNAPS_DIR/$VMID"
+
+    cd $VM_DIR
+    tar --transform="flags=r;s|${DISK_NAME}.snap/${SNAP_ID}|${DISK_NAME}.replica_snap|" \
+        -cSf - ${DISK_NAME}.snap/${SNAP_ID} | \
+        ssh $REPLICA_SSH_OPTS ${REPLICA_HOST} \
+        "tar xSf - -C $REPLICA_RECOVERY_SNAPS_DIR/$VMID"
+fi
 EOT
 )
 
-ssh_exec_and_log "${SRC_HOST}" "${CMD}" \
-                 "Error creating snapshot ${SNAP_PATH}"
+ssh_forward ssh_exec_and_log "${SRC_HOST}" "${CMD}" \
+    "Error creating snapshot ${SNAP_PATH}"
diff --git a/src/tm_mad/ssh/snap_create_live b/src/tm_mad/ssh/snap_create_live