diff --git a/README.md b/README.md index ce943e6..4fc3906 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,8 @@ The problem: 1 - manage xen server with simple commands 2 - new backup script +mdmxen - root dir, on deploy put it /root/bin/mdmxen +- lib - library dir +- etc - config dir, for backup config etc +- bin - shell scripts and other utils +/Users/mikedm/Projects/xen-tools/mdmxen/lib/fn_rotation.fsh diff --git a/mdmxen/bin/mdmxen_backuper.sh b/mdmxen/bin/mdmxen_backuper.sh new file mode 100644 index 0000000..d011f99 --- /dev/null +++ b/mdmxen/bin/mdmxen_backuper.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# mdmxen project - backuper for backup vm with days rotation etc +# version 0-test + +# include this host config +. /root/bin/mdmxen/etc/host.cfg +# include libraries for work +. /root/bin/mdmxen/lib/libsys.sh +. /root/bin/mdmxen/lib/librotation.sh +. /root/bin/mdmxen/lib/libxen.sh + +#main +putlog "---------- Start $ME ----------" +send_state "starting script" + +# lock file test +putlog "test lock file" +locktest_state +# test backup directory +putlog "backup target test" +hddtest +ltest_state "backup target test" +# create lock file +putlog "create lock file" +lockcreate +ltest_state "create lock file" +# clear tmp directory +putlog "clear tmp directory" +cleartmp +ltest_state "clear tmp directory" +# begin backup +putlog "begin backup" +send_state "on backup" + +send_state "run before actions" +putlog "run before action" +. /root/bin/mdmxen/etc/before.cfg + +send_state "run vms actions" +putlog "run vms action" + +. /root/bin/mdmxen/etc/vms.cfg + +# rotation +putlog "call rotation function" +# AMOUNT_OF_DAYS +send_state "on rotation" +day_rotation_state 3 + +send_state "run after actions" +putlog "run after action" +. /root/bin/mdmxen/etc/before.cfg + +putlog "delete lock file" +lockdelete +ltest_state "delete .lock file" +putlog "create ok file" +okcreate +ltest_state "create ok file" +send_state "done" +putlog "SUCCESS_BACKUP" +putlog "---------- finish $ME ----------" \ No newline at end of file diff --git a/mdmxen/etc/after.cfg b/mdmxen/etc/after.cfg new file mode 100644 index 0000000..f19819f --- /dev/null +++ b/mdmxen/etc/after.cfg @@ -0,0 +1 @@ +# mdmxen project = put here exec to run after backup \ No newline at end of file diff --git a/mdmxen/etc/before.cfg b/mdmxen/etc/before.cfg new file mode 100644 index 0000000..c61fae5 --- /dev/null +++ b/mdmxen/etc/before.cfg @@ -0,0 +1 @@ +# mdmxen project = put here exec to run before backup \ No newline at end of file diff --git a/mdmxen/etc/host.cfg b/mdmxen/etc/host.cfg new file mode 100644 index 0000000..1a32a39 --- /dev/null +++ b/mdmxen/etc/host.cfg @@ -0,0 +1,20 @@ +# mdmxen project = host configuration + +# state for monitoring +STATE_DIR="/var/tmp/backuper.state" +# log file +FLOG="/var/log/backup.log" +# current hostname +ME="XEN_ME" +# temporary directory for export vms +TMP_DIR="/mnt/backup/tmp" +# lock file +FLOCK="/var/run/backup.lock" +# flag file for backup volume +FBACK="/mnt/backup/iambackup.txt" +# directory for day rotation +DAYS_DIR="/mnt/backupxfs/days" +# current target dir +TARGET_DIR="/mnt/backupxfs/days/1" +# months rotation dir +MONTHS_DIR="/mnt/backupxfs/months" \ No newline at end of file diff --git a/mdmxen/etc/readme.md b/mdmxen/etc/readme.md new file mode 100644 index 0000000..f421517 --- /dev/null +++ b/mdmxen/etc/readme.md @@ -0,0 +1,4 @@ +host.cfg current host config +vms.cfg virtual machines list with backup configurations (format vmuuid vmname compress) +before.cfg tasks before backup +after.cfg tasks after backup, for example rsync time machine etc diff --git a/mdmxen/etc/vms.cfg b/mdmxen/etc/vms.cfg new file mode 100644 index 0000000..09a45d7 --- /dev/null +++ b/mdmxen/etc/vms.cfg @@ -0,0 +1,11 @@ +# mdmxen project = virtual machines list for backup +# +# backup vm run state from snapshot where UUID and file name and archive trigger (true false) +# backup_vm_run_snap UUID name true +# +# backup vm poweroff state +# backup_vm_off UUID name true +# +# backup vm poweroff state and snapshot created and vm running again. +# so backup from snapshot when poweroff state +# backup_vm_off_snap UUID name true \ No newline at end of file diff --git a/mdmxen/lib/librotation.sh b/mdmxen/lib/librotation.sh new file mode 100644 index 0000000..6df3bb2 --- /dev/null +++ b/mdmxen/lib/librotation.sh @@ -0,0 +1,290 @@ +# mdmxen project - library for backup rotation + +day_rotation(){ +AMOUNT_OF_DAYS=$1 +putlog "prepare archive" +putlog "remove old day - ${AMOUNT_OF_DAYS}" +rm -rf "$DAYS_DIR/${AMOUNT_OF_DAYS}" +ret=$? +if [[ $ret == "0" ]] + then putlog "--- SUCCESS remove" + else putlog "--- FAILURE remove. error code $ret. exit" ; exit 2 ; +fi + +tt="" +for (( c=$AMOUNT_OF_DAYS; c>=1; c-- )) +do + if [[ $c == $AMOUNT_OF_DAYS ]] + then + let "y=$c-1" + if ! [[ $y == "0" ]] + then tt=`echo "$y"` + fi + else + let "y=$c-1" + if ! [[ $y == "0" ]] + then tt=`echo "$tt $y"` + fi + fi +done + +for j in $tt +do + let "pp=$j + 1" + putlog "move day $j to $pp" + mv "$DAYS_DIR/$j" "$DAYS_DIR/$pp" + ret=$? + if [[ $ret == "0" ]] + then putlog "--- SUCCESS move dir $j to $pp" + else putlog "--- FAILURE move dir $j to $pp. error code $ret. exit"; exit 3 ; + fi +done + + +putlog "start create curren day dir" +mkdir -p $DAYS_DIR/1 +ret=$? +if [[ $ret == "0" ]] + then putlog "--- SUCCESS create current day dir" + else putlog "--- FAILURE create current day dir. error code $ret. exit"; exit 4; +fi + +putlog "move tmp to last day" +mv $TMP_DIR/* $TARGET_DIR/ +ltest $? +} + +month_rotation(){ +AMOUNT_OF_MONTH=$1 +putlog "test first day of month" +if [[ $(date +%d) == "01" ]] +then + putlog "first day of month. start month copy and move" + putlog "remove old dir" + rm -rf $MONTHS_DIR/${AMOUNT_OF_MONTH} + ret=$? + if [[ $ret == "0" ]] + then putlog "--- SUCCESS remmove dir ${AMOUNT_OF_MONTH}" + else putlog "--- FAILURE remmove dir ${AMOUNT_OF_MONTH}"; exit 27; + fi +tt="" +for (( c=$AMOUNT_OF_MONTH; c>=1; c-- )) +do + if [[ $c == $AMOUNT_OF_DAYS ]] + then + let "y=$c-1" + if ! [[ $y == "0" ]] + then tt=`echo "$y"` + fi + else + let "y=$c-1" + if ! [[ $y == "0" ]] + then tt=`echo "$tt $y"` + fi + fi +done + for j in $tt + do + let "pp=$j + 1" + putlog "move month $j to $pp" + mv "$MONTHS_DIR/$j" "$MONTHS_DIR/$pp" + ret=$? + if [[ $ret == "0" ]] + then putlog "--- SUCCESS move dir $j to $pp" + else putlog "--- FAILURE move dir $j to $pp. error code $ret. exit"; exit 7; + fi + done + putlog "create current month dir" + mkdir -p $MONTHS_DIR/1 + ret=$? + if [[ $ret == "0" ]] + then putlog "--- SUCCESS create current monts dir" + else putlog "--- FAILURE create current monts dir. error code $ret. exit"; exit 8; + fi + putlog "start copy current day to current month" + cp -r $DAYS_DIR/1/* $MONTHS_DIR/1/ + ret=$? + if [[ $ret == "0" ]] + then putlog "--- SUCCESS copy" + else putlog "--- FAILURE copy. error code $ret. exit" ; exit 9; + fi + putlog "stop month copy and move" +else + putlog "skip month copy and move" +fi + +} + +day_rotation_state(){ +DRS_AMOUNT_OF_DAYS=$1 +putlog "prepare archive" +putlog "remove old day - ${DRS_AMOUNT_OF_DAYS}" +rm -rf "$DAYS_DIR/${DRS_AMOUNT_OF_DAYS}" +ltest_state "remove old day - ${DRS_AMOUNT_OF_DAYS}" + +tt="" +for (( c=${DRS_AMOUNT_OF_DAYS}; c>=1; c-- )) +do + if [[ ${c} == ${DRS_AMOUNT_OF_DAYS} ]] + then + let "y=${c}-1" + if ! [[ ${y} == "0" ]] + then tt=`echo "${y}"` + fi + else + let "y=${c}-1" + if ! [[ ${y} == "0" ]] + then tt=`echo "${tt} ${y}"` + fi + fi +done + +for j in ${tt} +do + let "pp=${j} + 1" + putlog "move day ${j} to ${pp}" + mv "${DAYS_DIR}/${j}" "${DAYS_DIR}/${pp}" + ltest_state "move day ${j} to ${pp}" +done + + +putlog "start create curren day dir" +mkdir -p ${DAYS_DIR}/1 +ltest_state "start create curren day dir" +putlog "move tmp to last day" +mv ${TMP_DIR}/* ${TARGET_DIR}/ +ltest_state "move tmp to last day" +} + +month_rotation_state(){ +AMOUNT_OF_MONTH=$1 +putlog "test first day of month" +if [[ $(date +%d) == "01" ]] +then + putlog "first day of month. start month copy and move" + putlog "remove old dir" + rm -rf $MONTHS_DIR/${AMOUNT_OF_MONTH} + ltest_state "remove old dir" +tt="" +for (( c=$AMOUNT_OF_MONTH; c>=1; c-- )) +do + if [[ $c == $AMOUNT_OF_DAYS ]] + then + let "y=$c-1" + if ! [[ $y == "0" ]] + then tt=`echo "$y"` + fi + else + let "y=$c-1" + if ! [[ $y == "0" ]] + then tt=`echo "$tt $y"` + fi + fi +done + for j in $tt + do + let "pp=$j + 1" + putlog "move month $j to $pp" + mv "$MONTHS_DIR/$j" "$MONTHS_DIR/$pp" + ltest_state "move month $j to $pp" + done + putlog "create current month dir" + mkdir -p $MONTHS_DIR/1 + ltest_state "create current month dir" + putlog "start copy current day to current month" + cp -r $DAYS_DIR/1/* $MONTHS_DIR/1/ + ltest_state "start copy current day to current month" + putlog "stop month copy and move" +else + putlog "skip month copy and move" +fi + +} + +backuper_prepare_rotation_state(){ + #okfile + BPR_OKFILE="bone.txt" + #mkdir bin + MKDIR_BIN="/usr/bin/mkdir" + #mv bin + MV_BIN="/usr/bin/mv" + + #default status + BPR_DEF_FN_STATUS="0" + #current status + BPR_CUR_FN_STATUS="${BPR_DEF_FN_STATUS}" + if [[ ! ( -z ${1} && -z ${2} && -d ${1} && -d ${2} ) ]] + then + for BPR_CUR_DIR in `ls ${1}`; + do + if [[ -f ${1}/${BPR_CUR_DIR}/${BPR_OKFILE} ]] + then + putlog "Check directory: ${2}/${BPR_CUR_DIR}" + if [[ -d ${2}/${BPR_CUR_DIR} ]] + then + #skip + send_error_to_log "Directory ${2}/${BPR_CUR_DIR} exists. Skipping" + BPR_CUR_FN_STATUS="1" + else + #move + putlog "Make directory: ${2}/${BPR_CUR_DIR}" + ${MKDIR_BIN} ${2}/${BPR_CUR_DIR} + ltest_state "Mkdir ${BPR_CUR_DIR}" + + putlog "Moving files" + for BPR_CUR_FILE in `ls ${1}/${BPR_CUR_DIR}` + do + putlog "Move ${1}/${BPR_CUR_DIR}/${BPR_CUR_FILE}" + ${MV_BIN} ${1}/${BPR_CUR_DIR}/${BPR_CUR_FILE} ${2}/${BPR_CUR_DIR}/ + ltest_state "Move ${BPR_CUR_FILE}" + done + fi + else + #OKFILE NOT EXIST + putlog "${1}/${BPR_CUR_DIR}/${BPR_OKFILE} not exist. Skipping" + fi + done + fi + if [[ ${BPR_CUR_FN_STATUS} == ${BPR_DEF_FN_STATUS} ]] + then + [[ "1" == "1" ]] + else + [[ "1" == "0" ]] + fi +} + +backuper_rotation_days_state(){ + BRDS_DEF_FN_STATUS="0" + BRDS_CUR_FN_STATUS="${BRDS_DEF_FN_STATUS}" + + putlog "Start rotation" + send_state "on rotation" + + if [[ ! -z ${1} ]] + then + BRDS_AMOUNT_OF_DAYS=${1} + putlog "Start moving files from ${COL_TARGET}" + backuper_prepare_rotation_state "${COL_TARGET}" "${TMP_DIR}" + if [[ ${?} -ne "0" ]] + then + BRDS_CUR_FN_STATUS="1" + else + putlog "Moving files done" + + putlog "Rotate dirs..." + day_rotation_state ${BRDS_AMOUNT_OF_DAYS} + putlog "Finish rotate dirs" + fi + else + send_error_to_log "amount of days not set. skipping" + BRDS_CUR_FN_STATUS="1" + fi + + if [[ ${BRDS_CUR_FN_STATUS} == ${BRDS_DEF_FN_STATUS} ]] + then + [[ "1" == "1" ]] + else + [[ "1" == "0" ]] + fi + putlog "Finish rotation" +} diff --git a/mdmxen/lib/libsys.sh b/mdmxen/lib/libsys.sh new file mode 100644 index 0000000..d449b16 --- /dev/null +++ b/mdmxen/lib/libsys.sh @@ -0,0 +1,1358 @@ +# mdmxen project - library for system and basic algorithms + +#putlog +putlog(){ + echo "$(date +%Y-%m-%d\ %H:%M:%S ) $1" >> $FLOG +} + +#last op test +ltest(){ +if [[ $? == "0" ]] + then putlog "SUCCESS" ; + else putlog "FAILURE. error code $?. exit" ; exit 1; +fi +} + +#last op test huysnim =) +ltesth(){ +if [[ $? == "0" ]] + then putlog "SUCCESS" ; + else putlog "FAILURE. error code $?. skip exit" ; +fi +} + +#lock file test +locktest(){ +test -f $FLOCK +if [[ $? == "1" ]] + then putlog "SUCCESS" ; + else putlog "FAILURE"; exit 1; +fi +} + +#lock file test +locktest_state(){ +test -f $FLOCK +if [[ $? == "1" ]] + then putlog "SUCCESS" ; + else + putlog "FAILURE" + putlog "Send state to file" + echo "fail after - locktest" > $STATE_DIR + ltest + exit 1 +fi +} + +#test backup hdd +hddtest(){ +test -f $FBACK +} + +#create lock file +lockcreate(){ +touch $FLOCK +} + +#clear tmp folder +cleartmp(){ +rm -rf $TMP_DIR/* +} + +#delete lock file +lockdelete(){ +rm $FLOCK +} + +#create ok file +okcreate(){ +touch $TARGET_DIR/ok.txt +} + +#ltest with state +ltest_state(){ +if [[ $? == "0" ]] + then putlog "SUCCESS" ; + else + putlog "FAILURE. error code $?. exit" + putlog "Send state \"fail ${1}\" to file ${STATE_DIR}" + echo "fail $1" > $STATE_DIR + ltest + exit 1 +fi +} + +#get error msg from file ${1} and generate alert +ltest_state_continue(){ +if [[ ${?} -eq "0" ]] + then putlog "SUCCESS" ; + else + TAIL_BIN="/usr/bin/tail" + send_error_to_log "`cat ${1} | ${TAIL_BIN} -n 1`" +fi +} + +#check ip address +check_ip_valid(){ +#regular expression +#minimal valid ip: 1.0.0.1 +#maximal valid ip: 255.255.255.255 +IP_REGEXP="^([1-9][0-9]?|1[0-9]{2}|2[0-4][0-9]|25[0-5])(.(0|[1-9][0-9]?|1[0-9]{2}|2[0-4][0-9]|25[0-5])){2}.([1-9][0-9]?|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" + +[[ ( -z ${1} || "${1}" =~ ${IP_REGEXP} ) ]] +#after this function you need to use ltest* function +#0 - ip is valid +#1 - no parameter or invalid ip +} + +#check integer positive number +check_num_int_pos(){ +#minimum value: 1 +#maximum value: 65535 +REG='^([1-9]|[1-5][0-9]{1,4}|[6-9][0-9]{1,3}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' +[[ ( -z ${1} || ${1} =~ ${REG} ) ]] +} + +#send normal state func +send_state(){ +putlog "Send state \"${1}\" to file" +echo "$1" > $STATE_DIR +ltest +} + +#send fail state, wait and continue +create_alert(){ +#path to cat +CAT_BIN="/usr/bin/cat" +#path to sleep +SLEEP_BIN="/usr/bin/sleep" +#sleep amount +CRAL_ALERT_SLEEP="30" + + putlog "Start generating alert" + putlog "Get current backup state" + CRAL_CUR_STATE=`${CAT_BIN} ${STATE_DIR}` + send_state "$1" + putlog "Sleep: ${CRAL_ALERT_SLEEP}" + ${SLEEP_BIN} ${CRAL_ALERT_SLEEP} + putlog "Resuming backup. Recovering previous backup state" + send_state "${CRAL_CUR_STATE}" + [[ "1" == "0" ]] +} + +#send error message to log file and generate fail code 1 +send_error_to_log(){ + putlog "ERROR: ${1}" + [[ "1" == "0" ]] +} + +#get contents of nfs share +get_files_nfs4_state(){ +#default fn status +GFN4S_DEF_FN_STATUS="0" +#current fn status +GFN4S_CUR_FN_STATUS=${GFN4S_DEF_FN_STATUS} + +#path to rsync +RSYNC_BIN="/usr/bin/rsync" +#path to rm +RM_BIN="/usr/bin/rm" +#path to echo +ECHO_BIN="/usr/bin/echo" +#path to mount.nfs4 +MNTNFS4_BIN="/usr/sbin/mount.nfs4" +#path to umount +UMOUNT_BIN="/usr/bin/umount" +#path to cut bin +CUT_BIN="/usr/bin/cut" +#path to cat +CAT_BIN="/usr/bin/cat" +#path to cp +CP_BIN="/usr/bin/cp" +#path to mkdir +MKDIR_BIN="/usr/bin/mkdir" +#path to awk bin +AWK_BIN="/usr/bin/awk" +#path to head bin +HEAD_BIN="/usr/bin/head" +#path to tail bin +TAIL_BIN="/usr/bin/tail" +#path to sed bin +SED_BIN="/usr/bin/sed" +#date bin +DATE_BIN="/bin/date" +#stat bin +STAT_BIN="/usr/bin/stat" +#path to sleep +SLEEP_BIN="/usr/bin/sleep" +#sleep amount +GFN4S_SLEEP_AMOUNT="180" +#nfs source +GFN4S_NFS_SOURCE="/mnt/nfs" +#cache directory +GFN4S_NFSCACHE_DIR="/tmp" +#path to cache data (NAME WILL BE CHANGED IS SCRIPT BODY!!!) +GFN4S_NFSCACHE="${NFSCACHE_DIR}/default.nfscache" +#path to error data +GFN4S_NFSERROR="${NFSCACHE_DIR}/default.nfserror" +#default nfs port +GFN4S_NFS_PORT_DEFAULT="2049" +#ok file +GFN4S_OKFILE="ok.txt" + +#rsync bandwidth +#1200 ~ 100Mbit/s +GFN4S_DLIMIT="12000" +GFN4S_RSYNC_OPTS="--partial --bwlimit=${GFN4S_DLIMIT}" + +#check variables +putlog "check variables" +#1.check variables - start +if [[ ! (-z ${1} && -z ${2}) ]] +then + #1.1.variables set - start + GFN4S_SHARE_NAME=${2} + GFN4S_SERV_IP=${1} + #1.2.check parameter 3 - start + if [[ ! ( -z ${3} ) ]] + then + #1.2.1.parameter 3 exist - start + GFN4S_SERV_NAME=`${ECHO_BIN} ${3} | ${SED_BIN} -e 's/\.\.//g'` + #1.2.2.check serv name - start + if [[ "${GFN4S_SERV_NAME}" == "" ]] + then + #1.2.2.1.serv name invalid - start + #generate warning for monitoring + send_error_to_log "Server name empty. Use ip ${GFN4S_SERV_IP} as name" + GFN4S_SERV_NAME=${GFN4S_SERV_IP} + #1.2.2.1.serv name invalid - finish + fi + #1.2.2.check serv name - finish + #1.2.1.parameter 3 exist - finish + else + #1.2.3.parameter 3 not exist - start + GFN4S_SERV_NAME=${GFN4S_SERV_IP} + #1.2.3.parameter 3 not exist - finish + fi + #1.2.check parameter 3 - finish + putlog "Validate ip address" + check_ip_valid ${GFN4S_SERV_IP} + #check ip validation result + if [[ ! (${?} -eq "0") ]] + then + #1.1.1.ip invalid - start + send_error_to_log "IP: ${GFN4S_SERV_IP} invalid. Skipping..." + #generate fail exit code + GFN4S_CUR_FN_STATUS="1" + #1.1.1.ip infalid - finish + else + #1.1.2.ip valid - start + putlog "IP: ${GFN4S_SERV_IP} valid." + #checking share name + if [[ ! (${GFN4S_SHARE_NAME} == "") ]] + then + #1.1.2.0.check parameter 4 - start + if [[ ! ( -z ${4} ) ]] + then + #1.1.2.0.1.parameter 4 exist - start + GFN4S_NFS_PORT=${4} + #1.1.2.0.1.parameter 4 exist - finish + else + #1.1.2.0.2.parameter 4 not exist - start + GFN4S_NFS_PORT=${GFN4S_NFS_PORT_DEFAULT} + #1.1.2.0.2.parameter 4 not exist - finish + fi + #1.1.2.0.check parameter 4 - finish + #1.1.2.1.share name not empty - start + #mount nfs share + #variable not exist + #copy files by direct nfs (unsecure) + #mount share + #!!!!!!!!!! + #1.1.2.2.create caches for nfs - start + putlog "Create caches" + GFN4S_NFSCACHE="${GFN4S_NFSCACHE_DIR}/${GFN4S_SERV_NAME}.nfscache" + GFN4S_NFSERROR="${GFN4S_NFSCACHE_DIR}/${GFN4S_SERV_NAME}.nfserror" + putlog "Create: ${GFN4S_NFSCACHE}" + touch ${GFN4S_NFSCACHE} + ltest_state "Create: ${GFN4S_NFSCACHE}" + putlog "Create: ${GFN4S_NFSERROR}" + touch ${GFN4S_NFSERROR} + ltest_state "Create: ${GFN4S_NFSERROR}" + #1.1.2.2.create caches for nfs - finish + #STDOUT, STDERR, LSTATE + ${MNTNFS4_BIN} -o proto=tcp,port=${GFN4S_NFS_PORT} ${GFN4S_SERV_IP}:${GFN4S_SHARE_NAME} ${GFN4S_NFS_SOURCE} > ${GFN4S_NFSCACHE} 2> ${GFN4S_NFSERROR} + #1.1.2.3.check mount status - start + if [[ ${?} -ne "0" ]] + then + #1.1.2.3.1.mount error - start + #generate fail exit code for function + [[ "1" -eq "0" ]] + ltest_state_continue "${GFN4S_NFSERROR}" + #generate fail exit code again + GFN4S_CUR_FN_STATUS="1" + #1.1.2.3.1.mount error - finish + else + #1.1.2.3.2.mount success - start + #1.1.2.3.3.ok test - start + if [[ -f ${GFN4S_NFS_SOURCE}/${GFN4S_OKFILE} ]] + then + #1.1.2.3.3.1.ok file exist - start + putlog "Get current date" + GFN4S_CURDATE=`${DATE_BIN} +%Y-%m-%d` 2> ${GFN4S_NFSERROR} + ltest_state_continue "${GFN4S_NFSERROR}" + if [[ ${?} -ne "0" ]] + then + [[ "1" == "0" ]] + ltest_state "Get date" + fi + + putlog "Get modification date of file: ${GFN4S_NFS_SOURCE}/${GFN4S_OKFILE}" + GFN4S_FILEDATE=`${STAT_BIN} ${GFN4S_NFS_SOURCE}/${GFN4S_OKFILE} | ${GREP_BIN} 'Modify' | ${AWK_BIN} -F ' ' '{print \$2}'` 2> ${GFN4S_NFSERROR} + ltest_state_continue "${GFN4S_NFS_ERROR}" + #1.1.2.3.3.2.check dates - start + if [[ ${GFN4S_CURDATE} == ${GFN4S_FILEDATE} ]] + then + #1.1.2.3.3.2.1.dates equivalent - start + #1.1.2.3.3.2.2.check dir - start + if [[ ! -d ${TMP_DIR}/${GFN4S_SERV_NAME} ]] + then + #1.1.2.3.3.2.2.1.dir not exist - start + putlog "mkdir: ${TMP_DIR}/${GFN4S_SERV_NAME}" + ${MKDIR_BIN} -p ${TMP_DIR}/${GFN4S_SERV_NAME} > ${GFN4S_NFSCACHE} 2> ${GFN4S_NFSERROR} + #1.1.2.3.3.2.2.2.check mkdir - start + if [[ ${?} -ne "0" ]] + then + #1.1.2.3.3.2.2.2.1.mkdir fail - start + send_error_to_log "Fail to create dir: ${TMP_DIR}/${GFN4S_SERV_NAME}. Skipping share" + #generate fail exit code + [[ "1" -eq "0" ]] + ltest_state_continue "${GFN4S_NFSERROR}" + #generate fail exit code + GFN4S_CUR_FN_STATUS="1" + #1.1.2.3.3.2.2.2.1.mkdir fail - finish + else + #1.1.2.3.3.2.2.2.2.mkdir success - start + putlog "Start copy files" + for GFN4S_CURFILE in `ls ${GFN4S_NFS_SOURCE}/ | grep -v "${GFN4S_OKFILE}"`; + do + #1.1.2.3.3.2.2.2.2.1.file copy - start + putlog "Copy file: ${GFN4S_NFS_SOURCE}/${GFN4S_CURFILE}" + #copy - no speed limit, no append download after fail + #${CP_BIN} ${GFN4S_NFS_SOURCE}/${GFN4S_CURFILE} ${TMP_DIR}/${GFN4S_SERV_NAME}/ > ${GFN4S_NFSCACHE} 2> ${GFN4S_NFSERROR} + #rsync - speed limit, append download after fail + ${RSYNC_BIN} ${GFN4S_RSYNC_OPTS} ${GFN4S_NFS_SOURCE}/${GFN4S_CURFILE} ${TMP_DIR}/${GFN4S_SERV_NAME}/ > ${GFN4S_NFSCACHE} 2> ${GFN4S_NFSERROR} + ltest_state_continue "${GFN4S_NFSERROR}" + #set fail status + if [[ ${?} -ne "0" ]] + then + GFN4S_CUR_FN_STATUS="1" + fi + #1.1.2.3.3.2.2.2.2.1.file copy - finish + done + putlog "Copy file: ${GFN4S_NFS_SOURCE}/${GFN4S_OKFILE}" + ${CP_BIN} ${GFN4S_NFS_SOURCE}/${GFN4S_OKFILE} ${TMP_DIR}/${GFN4S_SERV_NAME}/ > ${GFN4S_NFSCACHE} 2> ${GFN4S_NFSERROR} + ltest_state_continue "${GFN4S_NFSERROR}" + if [[ ${?} -ne "0" ]] + then + GFN4S_CUR_FN_STATUS="1" + fi + putlog "Finish copy files" + #1.1.2.3.3.2.2.2.2.mkdir success - finish + fi + #1.1.2.3.3.2.2.2.check mkdir - finish + #1.1.2.3.3.2.2.1.dir not exist - finish + else + #1.1.2.3.3.2.2.3.dir exist - start + send_error_to_log "${TMP_DIR}/${GFN4S_SERV_NAME} exist. Skipping" + #generate fail exit code + GFN4S_CUR_FN_STATUS="1" + #1.1.2.3.3.2.2.3.dir exist - finish + fi + #1.1.2.3.3.2.2.check dir - finish + #1.1.2.3.3.2.1.dates equivalent - finish + else + #1.1.2.3.3.2.3.dates not equivalent - start + putlog "TODAY: ${GFN4S_CURDATE}; FILE DATE: ${GFN4S_FILEDATE}; skipping share" + send_error_to_log "backup in progress on remote share ${GFN4S_SERV_NAME}" + #generate fail exit code + GFN4S_CUR_FN_STATUS="1" + #1.1.2.3.3.2.3.dates not equivalent - finish + fi + #1.1.2.3.3.2.check dates - finish + #1.1.2.3.3.1.ok file exist - finish + else + #1.1.2.3.3.3.ok file not exist - start + putlog "${GFN4S_OKFILE} not exist. skipping share" + send_error_to_log "File ${GFN4S_OKFILE} not exist" + #generate fail exit code + GFN4S_CUR_FN_STATUS="1" + #1.1.2.3.3.3.ok file not exist - finish + fi + #1.1.2.3.3.ok test - finish + #umount share + #1.2.3.3.4.umount - start + putlog "UnMount ${GFN4S_NFS_SOURCE}" + ${UMOUNT_BIN} ${GFN4S_NFS_SOURCE} 2> ${GFN4S_NFSERROR} + ltest_state_continue "${GFN4S_NFSERROR}" + #если не получилось отмонтировать шару, то это очень нехорошая ситуация + #данный факт может оказать негативное влияние на будущие действия по резервированию + if [[ ${?} -ne "0" ]] + then + #если мы оказались здесь, то шару с первой попытки отмонтировать не получилось + #попробуем уснуть и повтороить попытку еще раз + putlog "Wait ${GFN4S_SLEEP_AMOUNT}s for second try..." + ${SLEEP_BIN} ${GFN4S_SLEEP_AMOUNT} 2> ${GFN4S_NFSERROR} + ltest_state_continue "${GFN4S_NFSERROR}" + putlog "UnMount ${GFN4S_NFS_SOURCE} - 2nd try..." + ${UMOUNT_BIN} ${GFN4S_NFS_SOURCE} 2> ${GFN4S_NFSERROR} + ltest_state_continue "${GFN4S_NFSERROR}" + #вторая попытка отмонтирования не помогла. ситуация скорее всего очень критичная + #тушим скрипт наглухо + ltest_state "umount ${GFN4S_NFS_SOURCE}" + fi + # + #1.1.2.3.4.umount - finish + #1.1.2.3.2.mount success - finish + fi + #1.1.2.3.check mount status - finish + #remove caches + putlog "Removing caches" + putlog "Remove: ${GFN4S_NFSCACHE}" + ${RM_BIN} -rf ${GFN4S_NFSCACHE} + ltest_state "rm ${GFN4S_NFSCACHE}" + putlog "Remove: ${GFN4S_NFSERROR}" + ${RM_BIN} -rf ${GFN4S_NFSERROR} + ltest_state "rm ${GFN4S_NFSERROR}" + #1.1.2.1.share name not empty - finish + else + #1.1.2.4.share name empty - start + putlog "Share name empty. Skipping share" + send_error_to_log "share name empty" + #generate fail exit code + GFN4S_CUR_FN_STATUS="1" + #1.1.2.4.share name empty - finish + fi + #1.1.2.ip valid - finish + fi + #1.1.variables set - finish +else + #1.2.variables not set - start + send_error_to_log "Variables not set. Skipping share" + #generate fail exit code + GFN4S_CUR_FN_STATUS="1" + #1.2.variables not set - finish +fi +#1.check variables - finish +#2.send fn status - start +if [[ ${GFN4S_CUR_FN_STATUS} == ${GFN4S_DEF_FN_STATUS} ]] +then + #2.1.status ok - start + [[ "1" == "1" ]] + #2.1.status ok - finish +else + #2.2.status fail - start + [[ "1" == "0" ]] + #2.2.status fail - finish +fi + +#2.send fn status - finish +} + +ssh_tunnel_check(){ + #default fn status + STCH_DEF_FN_STATUS="0" + #current fn status + STCH_CUR_FN_STATUS=${STCH_DEF_FN_STATUS} + + #system binary setup + PS_BIN="/usr/bin/ps" + GREP_BIN="/usr/bin/grep" + #1.check parameter - start + if [[ ! ( -z ${1} ) ]] + then + #1.1.parameter exist - start + STCH_RESULT=`${PS_BIN} faux | ${GREP_BIN} "${1}" | ${GREP_BIN} -v "${GREP_BIN}"` + if [[ ${?} -ne "0" ]] + then + STCH_CUR_FN_STATUS="1" + fi + #if grep has no lines then result code will be 1 - fail + #if grep has any lines then result code will be 0 - ok + #1.1.parameter exist - finish + else + #1.2.parameter not exist - start + #generate fail code + send_error_to_log "TunChk param.not.exist" + #запуск без параметра это критичный вызов. нужно глушить скрипт + ltest_state "TunChk no parameter" + #1.2.parameter not exist - finish + fi + #1.check parameter - finish + #2.send fn status - start + if [[ ${STCH_CUR_FN_STATUS} == ${STCH_DEF_FN_STATUS} ]] + then + #2.1.status ok - start + [[ "1" == "1" ]] + #2.1.status ok - finish + else + #2.2.status fail - start + [[ "1" == "0" ]] + #2.2.status fail - finish + fi + #2.send fn status - finish +} + +ssh_tunnel_create(){ + #path to echo + ECHO_BIN="/usr/bin/echo" + #path to date + DATE_BIN="/usr/bin/date" + #path to bc + BC_BIN="/usr/bin/bc" + #default fn status + STCR_DEF_FN_STATUS="0" + #current fn status + STCR_CUR_FN_STATUS=${STCR_DEF_FN_STATUS} + + SSH_BIN="/usr/bin/ssh" + #timeout value + STCR_SSH_OPTS="-o ConnectTimeout=20 -o PasswordAuthentication=no -o ChallengeResponseAuthentication=no" + STCR_SSH_TUNNEL_CACHE_DIR="/tmp" + STCR_SSH_TUNNEL_ERROR="${STCR_SSH_TUNNEL_CACHE_DIR}/ssh_stderr.cache" + STCR_SSH_TUNNEL_CACHE="${STCR_SSH_TUNNEL_CACHE_DIR}/ssh_stdout.cache" + #1.check parameter 1 - start + if [[ ! ( -z ${1} ) ]] + then + #1.1.parameter 1 exist - start + STCR_SERV_IP=${1} + putlog "Start check ip" + check_ip_valid ${STCR_SERV_IP} + if [[ ! ( ${?} -eq "0" ) ]] + then + #1.1.1.ip invalid - start + send_error_to_file "Host ip not valid. Skipping" + #generate fail last code + STCR_CUR_FN_STATUS="1" + #1.1.1.ip invalid - finish + else + #1.1.2.ip valid - start + #1.1.3.parameter 2 exist - start + if [[ ! ( -z ${2} ) ]] + then + #1.1.3.1.local port exist - start + STCR_LOCAL_PORT=${2} + #check valid value + check_num_int_pos ${STCR_LOCAL_PORT} + #1.1.3.2.local port check - start + if [[ ${?} -ne "0" ]] + then + #1.1.3.2.1.local port invalid - start + putlog "Local port invalid. Skipping" + send_error_to_log "Local port invalid ${STCR_LOCAL_PORT}" + #create false code for exit + STCR_CUR_FN_STATUS="1" + #1.1.3.2.1.local port invalid - finish + else + #1.1.3.2.1.local port valid - start + #1.1.3.2.2.parameter 3 check - start + if [[ ! ( -z ${3} ) ]] + then + #1.1.3.2.2.1.remote port exist - start + STCR_REMOTE_PORT=${3} + check_num_int_pos ${STCR_REMOTE_PORT} + #1.1.3.2.2.2.remote port check - start + if [[ ${?} -ne "0" ]] + then + #1.1.3.2.2.2.1.remote port invalid - start + send_error_to_log "Remote port not valid" + STCR_CUR_FN_STATUS="1" + #we created fail exit code + #1.1.3.2.2.2.1.remote port invalid - finih + fi + #1.1.3.2.2.2.remote port check - finish + #1.1.3.2.2.1.remote port exist - finish + else + #1.1.3.2.2.3.remote port not exist - start + STCR_REMOTE_PORT="2049" + #1.1.3.2.2.3.remote port not exist - finish + fi + #1.1.3.2.2.parameter 3 check - finish + #1.1.3.2.1.local port valid - finish + fi + #1.1.3.1.local port exist - finish + else + #1.1.3.3.local port not exist - start + STCR_LOCAL_PORT="2049" + STCR_REMOTE_PORT=${STCR_LOCAL_PORT} + #1.1.3.3.local port not exist - finish + fi + #1.1.3.parameter 2 exist - finish + # + #1.1.4.last operation check - start + if [[ ${?} -ne "0" ]] + then + #1.1.4.1.last operation fail - start + putlog "Skipped ssh tunnel creation procedure" + #create fail code again and exit from function + STCR_CUR_FN_STATUS="1" + #1.1.4.1.last operation fail - finish + else + #1.1.4.2.last operation ok - start + putlog "Checking if tunnel already exists: ${SSH_BIN} ${STCR_SSH_OPTS} -f -N -L ${STCR_LOCAL_PORT}:127.0.0.1:${STCR_REMOTE_PORT} root@${STCR_SERV_IP}" + ssh_tunnel_check "${SSH_BIN} ${STCR_SSH_OPTS} -f -N -L ${STCR_LOCAL_PORT}:127.0.0.1:${STCR_REMOTE_PORT} root@${STCR_SERV_IP}" + #1.1.4.3.check tunnel - start + if [[ ${?} -ne "0" ]] + then + #1.1.4.3.1.ssh tunnel not exist - start + #if we are here then SERV_IP, LOCAL_PORT && REMOTE_PORT - OK + #1.1.4.3.2.ssh tunnel create - start + putlog "Creating ssh tunnel..." + putlog "Try to establish connection: ${SSH_BIN} ${STCR_SSH_OPTS} -f -N -L ${STCR_LOCAL_PORT}:127.0.0.1:${STCR_REMOTE_PORT} root@${STCR_SERV_IP}" + ${SSH_BIN} ${STCR_SSH_OPTS} -f -N -L ${STCR_LOCAL_PORT}:127.0.0.1:${STCR_REMOTE_PORT} root@${STCR_SERV_IP} > ${STCR_SSH_TUNNEL_CACHE} 2> ${STCR_SSH_TUNNEL_ERROR} + ltest_state_continue "${STCR_SSH_TUNNEL_ERROR}" + if [[ ${?} -ne "0" ]] + then + STCR_CUR_FN_STATUS="1" + fi + #1.1.4.3.2.ssh tunnel create - finish + #1.1.4.3.1.ssh tunnel not exist - finish + else + #1.1.4.3.3.ssh tunnel exist - start + putlog "ssh tunnel already exist" + #данное событие нельзя однозначно трактовать как ошибку т.к., если туннель нужный нам существует, то можно сразу приступать к следующему этапу + #но пометуку в логе об этом сделать стоит + #1.1.4.3.3.ssh tunnel exist - finish + fi + #1.1.4.3.chechk tunnel - finish + #1.1.4.2.last operation ok - finish + fi + #1.1.4.last operation check - finish + fi + #1.1.parameter 1 exist - finish + else + #1.2.parameter 1 not exist - start + send_error_to_log "Host not set. Skipping" + #generate fail last code + STCR_CUR_FN_STATUS="1" + #1.2.parameter 1 not exist - finish + fi + #1.check parameter 1 - finish + #2.send fn state - start + if [[ ${STCR_CUR_FN_STATUS} == ${STCR_DEF_FN_STATUS} ]] + then + #2.1.status ok - start + [[ "1" == "1" ]] + #2.1.status ok - finish + else + #2.2.status fail - start + [[ "1" == "0" ]] + #2.2.status fail - finish + fi + #2.send fn state - finish +} + +process_kill(){ + #default function status + PKILL_DEF_FN_STATUS="0" + #current function status + PKILL_CUR_FN_STATUS=${PKILL_DEF_FN_STATUS} + #system binary setup + PS_BIN="/usr/bin/ps" + GREP_BIN="/usr/bin/grep" + KILL_BIN="/usr/bin/kill" + AWK_BIN="/usr/bin/awk" + #1.check parameter 1 - start + if [[ ! ( -z ${1} ) ]] + then + #1.1.parameter 1 exist - start + PKILL_RESULT=`${PS_BIN} faux | ${GREP_BIN} "${1}" | ${GREP_BIN} -v "${GREP_BIN}" | ${AWK_BIN} '{print $2}'` + if [[ "${PKILL_RESULT}" == "" ]] + then + #1.1.1.PID empty - start + send_error_to_log "Process not found. Skipping" + #generate fail exit code + PKILL_CUR_FN_STATUS="1" + #1.1.1.PID empty - finish + else + #1.1.2.PID not empty - start + putlog "Killing process with pid: ${PKILL_RESULT}" + ${KILL_BIN} -9 ${PKILL_RESULT} + if [[ ! ( ${?} -eq "0" ) ]] + then + #1.1.2.1.KILL fail - start + send_error_to_log "Kill process with pid ${PKILL_RESULT} fail" + #generate fail exit code + PKILL_CUR_FN_STATUS="1" + #1.1.2.1.KILL fail - finish + fi + #1.1.2.PID not empty - finish + fi + #1.1.parameter 1 exist - finish + else + #1.2.parameter 1 not exist - start + #generate fail code + PKILL_CUR_FN_STATUS="1" + #1.2.parameter 1 not exist - finish + fi + #1.check parameter 1 - finish + #2.send fn status - start + if [[ ${PKILL_CUR_FN_STATUS} == ${PKILL_DEF_FN_STATUS} ]] + then + #2.1.status ok - start + [[ "1" == "1" ]] + #2.1.status ok - finish + else + #2.2.status fail - start + [[ "1" == "0" ]] + #2.2.status fail - finish + fi + #2.send fn status - finish +} + +ssh_tunnel_destroy(){ +#ssh opts +STDR_SSH_OPTS="-o ConnectTimeout=20 -o PasswordAuthentication=no -o ChallengeResponseAuthentication=no" +#default function status +STDR_DEF_FN_STATUS="0" +#current function status +STDR_CUR_FN_STATUS=${STDR_DEF_FN_STATUS} + + SSH_BIN="/usr/bin/ssh" + STDR_SSH_TUNNEL_CACHE_DIR="/tmp" + STDR_SSH_TUNNEL_ERROR="${SSH_TUNNEL_CACHE_DIR}/ssh_stderr.cache" + STDR_SSH_TUNNEL_CACHE="${SSH_TUNNEL_CACHE_DIR}/ssh_stdout.cache" + #1.check parameter 1 - start + if [[ ! ( -z ${1} ) ]] + then + #1.1.parameter 1 exist - start + STDR_SERV_IP=${1} + putlog "Start check ip" + #1.2.check ip - start + check_ip_valid ${STDR_SERV_IP} + if [[ ! ( ${?} -eq "0" ) ]] + then + #1.2.1.ip invalid - start + send_error_to_log "Host ip not valid. Skipping" + #generate fail last code + STDR_CUR_FN_STATUS="1" + #1.2.1.ip invalid - finish + else + #1.2.2.ip valid - start + #1.2.3.check parameter 2 - start + if [[ ! ( -z ${2} ) ]] + then + #1.2.3.1.parameter 2 exist - start + STDR_LOCAL_PORT=${2} + #1.2.3.2.local port check - start + check_num_int_pos ${STDR_LOCAL_PORT} + if [[ ${?} -ne "0" ]] + then + #1.2.3.2.1.local port invalid - start + send_error_to_log "Local port invalid" + #create false code for exit + STDR_CUR_FN_STATUS="1" + #1.2.3.2.1.local port invalid - finish + else + #1.2.3.2.2.local port valid - start + #1.2.3.2.3.check parameter 3 - start + if [[ ! ( -z ${3} ) ]] + then + #1.2.3.2.3.1.parameter 3 exist - start + STDR_REMOTE_PORT=${3} + #1.2.3.2.3.2.remote port check - start + check_num_int_pos ${STDR_REMOTE_PORT} + if [[ ${?} -ne "0" ]] + then + #1.2.3.2.3.2.1.remote port invalid - start + send_error_to_log "Remote port not valid. Skipping" + STDR_CUR_FN_STATUS="1" + #we created fail exit code + #1.2.3.2.3.2.1.remote port invalid - finish + fi + #1.2.3.2.3.2.remote port check - fnish + #1.2.3.2.3.1.parameter 3 exist - finish + else + #1.2.3.2.3.3.remote port not exist - start + STDR_REMOTE_PORT="2049" + #1.2.3.2.3.3.remote port not exist - finish + fi + #1.2.3.2.3.check paraeter 3 - finish + #1.2.3.2.2.local port valid - finish + fi + #1.2.3.2.local port check - finish + #1.2.3.1.parameter 2 exist - finish + else + #1.2.3.3.parameter 2 not exist - start + STDR_LOCAL_PORT="2049" + STDR_REMOTE_PORT=${STDR_LOCAL_PORT} + #1.2.3.3.parameter 3 not exist - finish + fi + #1.2.3.check parameter 2 - finish + + #1.2.4.last operation status - start + if [[ ${?} -ne "0" ]] + then + #1.2.4.1.last operation fail - start + putlog "Skipped ssh tunnel creation procedure" + #create fail code again and exit from function + STDR_CUR_FN_STATUS="1" + #1.2.4.1.last operation fail - finish + else + #1.2.4.2.last operation ok - start + putlog "Checking if tunnel already exists: ${SSH_BIN} ${STDR_SSH_OPTS} -f -N -L ${STDR_LOCAL_PORT}:127.0.0.1:${STDR_REMOTE_PORT} root@${STDR_SERV_IP}" + #1.2.4.3.ssh tunnel check - start + ssh_tunnel_check "${SSH_BIN} ${STDR_SSH_OPTS} -f -N -L ${STDR_LOCAL_PORT}:127.0.0.1:${STDR_REMOTE_PORT} root@${STDR_SERV_IP}" + if [[ ${?} -eq "0" ]] + then + #1.2.4.3.1.ssh tunnel exist - start + #if we are here then STDR_SERV_IP, STDR_LOCAL_PORT && STDR_REMOTE_PORT - OK + #1.2.4.3.2.ssh tunnel destroy - start + putlog "Destroing ssh tunnel..." + process_kill "${SSH_BIN} ${STDR_SSH_OPTS} -f -N -L ${STDR_LOCAL_PORT}:127.0.0.1:${STDR_REMOTE_PORT} root@${STDR_SERV_IP}" + ltest_state "Destroy ssh tunnel" + #ошибка отказа при убийстве туннеля может критично повлиять на грядущие бэкапы + #в связи с этим данную ошибку нельзя игнорировать и нужно прервать выполнение скрипта + #1.2.4.3.2.ssh tunnel destroy - finish + #1.2.4.3.1.ssh tunnel exist - finish + else + #1.1.2.4.3.ssh tunnel not exist - start + putlog "ssh tunnel not exist" + #возможно туннель упал сам к этому моменту + #делаю пометку в лог файл. трактовать это как ошибку не совсем корректно + #1.1.2.4.3.ssh tunnel not exist - finish + fi + #1.2.4.2.last operation ok - finish + fi + #1.2.4.last operation status - finish + fi + #1.2.check ip - finish + #1.1.parameter 1 exist - finish + else + #1.3.parameter 1 not exist - start + send_error_to_log "Host ip not set. Skipping" + #generate fail last code + STDR_CUR_FN_STATUS="1" + #1.3.parameter 1 not exist - finish + fi + #1.check parameter 1 - finish + #2.send fn status - start + if [[ ${STDR_CUR_FN_STATUS} == ${STDR_DEF_FN_STATUS} ]] + then + #2.1.status ok - start + #return 0 + [[ "1" == "1" ]] + #2.1.status ok - finish + else + #2.2.status fail - start + #return 1 + [[ "1" == "0" ]] + #2.2.status fail - finish + fi + #2.send fn status - finish +} + +get_files_nfs4_ssh_state(){ +#check variables +#{1} - server ip +#{2} - share path +#{3} - local port for ssh +#{4} - remote port for ssh +#{5} - server name +#default function status +GFN4SS_DEF_FN_STATUS="0" +#current function status +GFN4SS_CUR_FN_STATUS=${GFN4SS_DEF_FN_STATUS} +#default nfs port +GFN4SS_NFS_PORT_DEFAULT="2049" + #1. check SERV_IP variable exist - start + if [[ ! ( -z ${1} ) ]] + then + #1.1 SERV_IP set - start + GFN4SS_SERV_IP=${1} + check_ip_valid ${GFN4SS_SERV_IP} + if [[ ! ( ${?} -eq "0" ) ]] + then + #1.1.1.SERV_IP not valid - start + send_error_to_log "Server ip not valid. Skipping" + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #1.1.1.SERV_IP not valid - finish + else + #1.1.2.SERV_IP valid - start + if [[ -z ${2} ]] + then + #1.1.2.1.SHARE_NAME not set - start + send_error_to_log "Share name not set. Skipping" + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #1.1.2.1.SHARE_NAME not set - finish + else + #1.1.2.2.SHARE_NAME set - start + GFN4SS_SHARE_NAME=${2} + if [[ ${GFN4SS_SHARE_NAME} == "" ]] + then + #1.1.2.2.1.SHARE_NAME empty - start + send_error_to_log "Share name empty. Skipping" + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #1.1.2.2.1.SHARE_NAME empty - finish + else + #1.1.2.2.2.SHARE_NAME not empty - start + if [[ -z ${4} ]] + then + #1.1.2.2.2.1.LOCAL_PORT not exist - start + GFN4SS_LOCAL_PORT="2049" + GFN4SS_REMOTE_PORT=${GFN4SS_LOCAL_PORT} + #1.1.2.2.2.1.LOCAL_PORT not exist - finish + else + #1.1.2.2.2.2.LOCAL_PORT exist - start + GFN4SS_LOCAL_PORT=${4} + check_num_int_pos ${GFN4SS_LOCAL_PORT} + if [[ ! ( ${?} -eq "0" ) ]] + then + #1.1.2.2.2.2.1.LOCAL_PORT invalid - start + send_error_to_log "Local port invalid. Skipping" + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #1.1.2.2.2.2.1.LOCAL_PORT invalid - finish + else + #1.1.2.2.2.2.2.LOCAL_PORT valid - start + if [[ -z ${5} ]] + then + #1.1.2.2.2.2.2.1.REMOTE_PORT not exist - start + GFN4SS_REMOTE_PORT=${GFN4SS_NFS_PORT_DEFAULT} + #1.1.2.2.2.2.2.1.REMOTE_PORT not exist - finish + else + #1.1.2.2.2.2.2.2.REMOTE_PORT exist - start + GFN4SS_REMOTE_PORT=${5} + check_num_int_pos ${GFN4SS_REMOTE_PORT} + if [[ ! ( ${?} -eq "0" ) ]] + then + #1.1.2.2.2.2.2.2.1.REMOTE_PORT invalid - start + send_error_to_log "Remote port invalid. Skipping" + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #1.1.2.2.2.2.2.2.1.REMOTE_PORT invalid - finish + fi + #1.1.2.2.2.2.2.2.REMOTE_PORT exist - finish + fi + #1.1.2.2.2.2.2.LOCAL_PORT valid - finish + fi + #1.1.2.2.2.2.LOCAL_PORT exist - finish + fi + #1.1.2.2.2.SHARE_NAME not empty - finish + #1.1.2.2.3.check parameter 5 - start + if [[ ! ( -z ${3} ) ]] + then + #1.1.2.2.3.1.parameter 5 exist - start + GFN4SS_SERV_NAME=${3} + #1.1.2.2.3.1.parameter 5 exist - finish + else + #1.1.2.2.3.2.parameter 5 not exist - start + GFN4SS_SERV_NAME=${GFN4SS_SERV_IP} + #1.1.2.2.3.2.parameter 5 not exist - finish + fi + #1.1.2.2.3.check parameter 5 - finish + fi + #1.1.2.2.SHARE_NAME set - finish + fi + #1.1.2.SERV_IP valid - finish + fi + #1.1.SERV_IP set - finish + else + #1.2.SERV_IP not set - start + send_error_to_log "Server ip not set. Skipping" + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #1.2.SERV_IP not set - finish + fi + #1. check SERV_IP variable exist - finish + # + #2. check 1. step status - start + #if 0 then - ok + #if 1 then - variables settings error - fail - exit + if [[ ${GFN4SS_CUR_FN_STATUS} == ${GFN4SS_DEF_FN_STATUS} ]] + then + #2.1.variable settings status - ok - start + #create ssh tunnel + #putlog "exec: ssh_tunnel_create ${GFN4SS_SERV_IP} ${GFN4SS_LOCAL_PORT} ${GFN4SS_REMOTE_PORT}" + ssh_tunnel_create ${GFN4SS_SERV_IP} ${GFN4SS_LOCAL_PORT} ${GFN4SS_REMOTE_PORT} + if [[ ! ( ${?} -eq "0" ) ]] + then + #2.1.1.ssh tunnel create fail - start + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #2.1.1.ssh tunnel create fail - finish + else + #2.1.2.ssh tunnel create ok - start + #execute get_files_nfs4_state() + get_files_nfs4_state "127.0.0.1" ${GFN4SS_SHARE_NAME} ${GFN4SS_SERV_NAME} ${GFN4SS_LOCAL_PORT} + #check fn status + if [[ ${?} -ne "0" ]] + then + #функция выполнилась с нефаталными ошибками + GFN4SS_CUR_FN_STATUS="1" + fi + #destroy ssh tunnel + #putlog "Destroy tunnel: ssh_tunnel_destroy ${GFN4SS_SERV_IP} ${GFN4SS_LOCAL_PORT} ${GFN4SS_REMOTE_PORT}" + ssh_tunnel_destroy ${GFN4SS_SERV_IP} ${GFN4SS_LOCAL_PORT} ${GFN4SS_REMOTE_PORT} + if [[ ! ( ${?} -eq "0" ) ]] + then + #2.1.2.1.ssh tunnel destroy fail - start + send_error_to_log "Fail to destroy ssh tunnel" + #generate fail exit code + GFN4SS_CUR_FN_STATUS="1" + #2.1.2.1.ssh tunnel destroy fail - finish + fi + #2.1.2.ssh tunnel create ok - finish + fi + #2.1.variable settings status - ok - finish + fi + #2. check 1. step status - finish + #3. send FN status - start + if [[ ${GFN4SS_CUR_FN_STATUS} == ${GFN4SS_DEF_FN_STATUS} ]] + then + #3.1.status ok - start + #return 0 + [[ "1" == "1" ]] + #3.1.status ok - finish + else + #3.2.status fail - start + #return 1 + [[ "1" == "0" ]] + #3.2.status fail - finish + fi + #3. send FN status - finish +} + +#get contents of smb share +get_files_smb_state(){ +#default fn status +GFSS_DEF_FN_STATUS="0" +#current fn status +GFSS_CUR_FN_STATUS=${GFSS_DEF_FN_STATUS} + +#path to rm +RM_BIN="/usr/bin/rm" +#path to echo +ECHO_BIN="/usr/bin/echo" +#path to cat +CAT_BIN="/usr/bin/cat" +#path to wc +WC_BIN="/usr/bin/wc" +#path to grep +GREP_BIN="/usr/bin/grep" +#path to mkdir +MKDIR_BIN="/usr/bin/mkdir" +#path to smbclient +SMBCLIENT_BIN="/usr/bin/smbclient" +#path to cut bin +CUT_BIN="/usr/bin/cut" +#path to awk bin +AWK_BIN="/usr/bin/awk" +#path to head bin +HEAD_BIN="/usr/bin/head" +#path to tail bin +TAIL_BIN="/usr/bin/tail" +#path to sed bin +SED_BIN="/usr/bin/sed" +#cache directory +GFSS_SMBCACHE_DIR="/tmp" +#path to cache data (NAME WILL BE CHANGED IS SCRIPT BODY!!!) +GFSS_SMBCACHE="${SMBCACHE_DIR}/default.smbcache" +#path to error data +GFSS_SMBERROR="${SMBCACHE_DIR}/default.smberror" +#path to ok-file +GFSS_OKFILE="bone.txt" + +#check variables +putlog "check variables" +if [[ ! (-z ${1} && -z ${2} && -z ${3}) ]] +then + #all variables are set (ok) + GFSS_SHARE_NAME=`${ECHO_BIN} ${2} | ${CUT_BIN} -d '_' -f 2` + GFSS_SERV_IP=${1} + GFSS_USER_NAME=${2} + GFSS_PASS=${3} + putlog "Validate ip address" + check_ip_valid ${GFSS_SERV_IP} + #check ip validation result + if [[ ! (${?} -eq "0") ]] + then + #invalid ip address + send_error_to_log "IP: ${GFSS_SERV_IP} invalid. Skipping..." + GFSS_CUR_FN_STATUS="1" + else + #valid ip address + putlog "IP: ${GFSS_SERV_IP} valid." + #checking share name + if [[ ! (${GFSS_SHARE_NAME} == "") ]] + then + #share name not empty string + #we need to create data cache file and check connection + putlog "Creating caches: generate filenames" + GFSS_SMBCACHE="${GFSS_SMBCACHE_DIR}/`${ECHO_BIN} ${GFSS_SHARE_NAME} | ${SED_BIN} -e 's/\\.\\.//g'`.smbcache" + if [[ ${?} -ne "0" ]] + then + GFSS_CUR_FN_STATUS="1" + fi + #and create file with stderr messages + GFSS_SMBERROR="${GFSS_SMBCACHE_DIR}/`${ECHO_BIN} ${GFSS_SHARE_NAME} | ${SED_BIN} -e 's/\\.\\.//g'`.smberror" + if [[ ${?} -ne "0" ]] + then + GFSS_CUR_FN_STATUS="1" + fi + #промежуточная проверка. если на этом этапе возникла ошибка, то она связана только с процессом генерации имен + #временных файлов. неправильно сгенерированные имена файлов в процессе работы скрипта могут привести к непредвиденным + #последствиям. по-этому, перестраховываюсь и прекращаю работу скрипта + if [[ ${GFSS_CUR_FN_STATUS} != ${GFSS_DEF_FN_STATUS} ]] + then + [[ "1" == "0" ]] + fi + ltest_state "generate filenames" + #if we cannot create file we must stop the script and send state + putlog "Create file: ${GFSS_SMBCACHE}" + touch ${GFSS_SMBCACHE} + ltest_state "create ${GFSS_SMBCACHE}" + + putlog "Create file: ${GFSS_SMBERROR}" + touch ${GFSS_SMBERROR} + ltest_state "create ${GFSS_SMBERROR}" + #touch это локальная операция. если не получилось записать в файловую систему, то + #велика вероятность того что все последующие операции резервирования шар будут неудачны + #в свзяи с этим тушим скрипт окончательно + #аналогично для файла с ошибками + putlog "Get contents of ${GFSS_SHARE_NAME} share" + ${SMBCLIENT_BIN} -e //${GFSS_SERV_IP}/${GFSS_SHARE_NAME} -U ${GFSS_USER_NAME}\%${GFSS_PASS} -I ${GFSS_SERV_IP} -c "ls" > ${GFSS_SMBCACHE} 2> ${GFSS_SMBERROR} + #if we cannot get contents we need to: + #1) create alert with error + #2) wait some seconds + #3) skip and continue with other share + if [[ ${?} -ne "0" ]] + then + #we got some error + #errors in smbclient go through STDOUT + #warnings in smbclient go through STDERR + [[ "1" == "0" ]] + ltest_state_continue "${GFSS_SMBCACHE}" + GFSS_CUR_FN_STATUS="1" + else + #ok + #check smbcache + #if wc -l > 0 and bone exist - ok + #if wc -l > 0 and bone ! exist - warn + #if wc -l = 0 skip + ${CAT_BIN} ${GFSS_SMBCACHE} | ${TAIL_BIN} -n +3 | ${HEAD_BIN} -n -2 | ${GREP_BIN} -e ".*" > /dev/null 2>&1 + #if [ ! -s ${GFSS_SMBCACHE} ] + #if [[ $( ${CAT_BIN} ${GFSS_SMBCACHE} | ${WC_BIN} -l ) -eq 0 ]] + if [[ ${?} -eq "1" ]] + then + #empty share + putlog "empty share. skip" + else + #share not empty + ${CAT_BIN} ${GFSS_SMBCACHE} | ${GREP_BIN} ${GFSS_OKFILE} > ${GFSS_SMBERROR} 2>&1 + if [[ ${?} -eq "0" ]] + then + #bone exist. copy files + #нужно проверить существует ли уже каталог + if [[ ! -d ${TMP_DIR}/${ARCH}/${GFSS_SHARE_NAME} ]] + then + #каталога не существует. создаем и резервируемся + putlog "mkdir: ${TMP_DIR}/${GFSS_SHARE_NAME}" + ${MKDIR_BIN} -p ${TMP_DIR}/${GFSS_SHARE_NAME} + ltest_state "mkdir: ${TMP_DIR}/${GFSS_SHARE_NAME}" + + putlog "start copy files" + for GFSS_CURFILE in `${CAT_BIN} ${GFSS_SMBCACHE} | ${TAIL_BIN} -n +3 | ${HEAD_BIN} -n -2 | ${AWK_BIN} '{print \$1}'`; + do + putlog "copy file: ${GFSS_CURFILE}" + ${SMBCLIENT_BIN} -e //${GFSS_SERV_IP}/${GFSS_SHARE_NAME} -U ${GFSS_USER_NAME}\%${GFSS_PASS} -I ${GFSS_SERV_IP} -c "get ${GFSS_CURFILE} ${TMP_DIR}/${GFSS_SHARE_NAME}/${GFSS_CURFILE}" > ${GFSS_SMBERROR} 2>1 + ltest_state_continue "${GFSS_SMBERROR}" + if [[ ${?} -ne "0" ]] + then + GFSS_CUR_FN_STATUS="1" + fi + done + putlog "copy files done" + else + #каталог уже существует + send_error_to_log "${TMP_DIR}/${GFSS_SHARE_NAME} exist. Skipping" + GFSS_CUR_FN_STATUS="1" + fi + else + #files exist but bone.txt not exist + #generate alert sleep and skip + send_error_to_log "Backup is in progress on remote share ${GFSS_SHARE_NAME}" + GFSS_CUR_FN_STATUS="1" + fi + fi + fi + putlog "Destroy ${GFSS_SMBCACHE}" + ${RM_BIN} -rf ${GFSS_SMBCACHE} + ltest_state "rm ${GFSS_SMBCACHE}" + putlog "Destroy ${GFSS_SMBERROR}" + ${RM_BIN} -rf ${GFSS_SMBERROR} + ltest_state "rm ${GFSS_SMBERROR}" + else + #share name empty string + putlog "share name empty. skipping" + GFSS_CUR_FN_STATUS="1" + fi + fi +else + #one or more variables not exist + send_error_to_log "Ip addr or/and share name or/and password not set" + GFSS_CUR_FN_STATUS="1" +fi +#2.send fn status - start +if [[ ${GFSS_CUR_FN_STATUS} == ${GFSS_DEF_FN_STATUS} ]] +then + #2.1.status ok - start + [[ "1" == "1" ]] + #2.1.status ok - finish +else + #2.2.status fail - start + [[ "1" == "0" ]] + #2.2.status fail - finish +fi +# +#2.send fn status - finish +} + +get_nfs_backups_state(){ +#path to cat +CAT_BIN="/usr/bin/cat" +#path to cut +CUT_BIN="/usr/bin/cut" +#path to grep +GREP_BIN="/usr/bin/grep" +#path to wc +WC_BIN="/usr/bin/wc" + +GNB_DEF_FN_STATUS="0" +GNB_CUR_FN_STATUS="${GNB_DEF_FN_STATUS}" + +if [[ -f ${1} ]] +then + #file exists + if [[ $( ${CAT_BIN} ${1} | ${GREP_BIN} -v -e '^$' | ${WC_BIN} -l ) -ne 0 ]] + then + #exist lines + putlog "Start backup NFS shares" + for CUR_NFS_SHARE in `cat ${NFS_BASE}` + do + #parsing file + CUR_HOST=`echo ${CUR_NFS_SHARE} | ${CUT_BIN} -d ';' -f 1` + CUR_SHARE=`echo ${CUR_NFS_SHARE} | ${CUT_BIN} -d ';' -f 2` + CUR_SERV_NAME=`echo ${CUR_NFS_SHARE} | ${CUT_BIN} -d ';' -f3` + CUR_LPORT=`echo ${CUR_NFS_SHARE} | ${CUT_BIN} -d ';' -f 4` + CUR_RPORT=`echo ${CUR_NFS_SHARE} | ${CUT_BIN} -d ';' -f 5` + #execute backup + putlog "Try to backup ${CUR_HOST}:${CUR_SHARE}" + #putlog "Cur lport: ${CUR_LPORT}; Cur rport: ${CUR_RPORT}" + if [[ ! ( -z ${CUR_LPORT} ) ]] + then + #lport defined + if [[ ! ( -z ${CUR_RPORT} ) ]] + then + #rport defined + #putlog "exec: get_files_nfs4_ssh_state ${CUR_HOST} ${CUR_SHARE} ${CUR_SERV_NAME} ${CUR_LPORT} ${CUR_RPORT}" + get_files_nfs4_ssh_state ${CUR_HOST} ${CUR_SHARE} ${CUR_SERV_NAME} ${CUR_LPORT} ${CUR_RPORT} + else + #rport not defined + #putlog "exec: get_files_nfs4_ssh_state ${CUR_HOST} ${CUR_SHARE} ${CUR_SERV_NAME} ${CUR_LPORT}" + get_files_nfs4_ssh_state ${CUR_HOST} ${CUR_SHARE} ${CUR_SERV_NAME} ${CUR_LPORT} + fi + else + #lport not defined + #putlog "exec: get_files_nfs4_ssh_state ${CUR_HOST} ${CUR_SHARE} ${CUR_SERV_NAME}" + get_files_nfs4_ssh_state ${CUR_HOST} ${CUR_SHARE} ${CUR_SERV_NAME} + fi + #get_files_smb_state ${CUR_HOST} ${CUR_USER} ${CUR_PASS} + if [[ ${?} -ne "0" ]] + then + #изменить статус, если хотя бы один вызов функции выполнился с некритичными ошибками + GNB_CUR_FN_STATUS="1" + fi + done + putlog "Finish backup nfs shares" + else + send_error_to_log "no valid lines in ${NFS_BASE}. skip nfs" + GNB_CUR_FN_STATUS="1" + fi +else + send_error_to_log "File not exist: ${1}" + GNB_CUR_FN_STATUS="1" +fi + +if [[ ${GNB_CUR_FN_STATUS} == ${GNB_DEF_FN_STATUS} ]] +then + #2.1.status ok - start + [[ "1" == "1" ]] + #2.1.status ok - finish +else + #2.2.status fail - start + [[ "1" == "0" ]] + #2.2.status fail - finish +fi +} + +get_smb_backups_state(){ +#path to cat +CAT_BIN="/usr/bin/cat" +#path to cut +CUT_BIN="/usr/bin/cut" +#path to grep +GREP_BIN="/usr/bin/grep" +#path to wc +WC_BIN="/usr/bin/wc" + +GSB_DEF_FN_STATUS="0" +GSB_CUR_FN_STATUS="${GSB_DEF_FN_STATUS}" + +if [[ -f ${1} ]] +then + #file exists + if [[ $( ${CAT_BIN} ${SMB_BASE} | ${GREP_BIN} -v -e '^$' | ${WC_BIN} -l ) -ne 0 ]] + then + #exist lines + putlog "Start backup smb shares" + for CUR_SMB_SHARE in `cat ${SMB_BASE}` + do + #parsing file + CUR_HOST=`echo ${CUR_SMB_SHARE} | ${CUT_BIN} -d ';' -f 1` + CUR_USER=`echo ${CUR_SMB_SHARE} | ${CUT_BIN} -d ';' -f 2` + CUR_PASS=`echo ${CUR_SMB_SHARE} | ${CUT_BIN} -d ';' -f 3` + #execute backup + putlog "Try to backup `echo ${CUR_USER} | ${CUT_BIN} -d '_' -f 2`" + get_files_smb_state ${CUR_HOST} ${CUR_USER} ${CUR_PASS} + if [[ ${?} -ne "0" ]] + then + #изменить статус, если хотя бы один вызов функции выполнился с некритичными ошибками + GSB_CUR_FN_STATUS="1" + fi + done + putlog "Finish backup smb shares" + else + putlog "no valid lines in ${SMB_BASE}. skip smb" + GSB_CUR_FN_STATUS="1" + fi +else + #smb_base not exist + putlog "no smb_base file. skip smb" + GSB_CUR_FN_STATUS="1" +fi + +if [[ ${GSB_CUR_FN_STATUS} == ${GSB_DEF_FN_STATUS} ]] +then + #2.1.status ok - start + [[ "1" == "1" ]] + #2.1.status ok - finish +else + #2.2.status fail - start + [[ "1" == "0" ]] + #2.2.status fail - finish +fi +} diff --git a/mdmxen/lib/libxen.sh b/mdmxen/lib/libxen.sh new file mode 100644 index 0000000..992ec0d --- /dev/null +++ b/mdmxen/lib/libxen.sh @@ -0,0 +1,82 @@ +# mdmxen project - library for xen + +backup_vm_run_snap(){ + VM_UUID=$1 + VM_NAME=$2 + COMPRESS=$3 + putlog "start backup_vpm_run_snap $VM_NAME" + + putlog "create snapshot" + SNAP_UUID=$(xe vm-snapshot uuid=$VM_UUID new-name-label=snap-$VM_NAME) + ltest_state "create snapshot" + + putlog "prepare export" + xe template-param-set is-a-template=false ha-always-run=false uuid=$SNAP_UUID + ltest_state "prepare export" + + putlog "export vm" + xe vm-export vm=$SNAP_UUID filename=$TMP_DIR/$VM_NAME.xva compress=$COMPRESS + ltest_state "export vm" + + putlog "delete snapshot" + xe vm-uninstall uuid=$SNAP_UUID force=true + ltest_state "delete snapshot" + putlog "stop backup_vpm_run_snap $VM_NAME" +} + +backup_vm_off(){ + VM_UUID=$1 + VM_NAME=$2 + COMPRESS=$3 + putlog "start backup_vm_off $VM_NAME" + + putlog "shutdown vm $VM_NAME" + xe vm-shutdown vm=$VM_UUID + ltest_state "shutdown vm $VM_NAME" + sleep 60 + + putlog "export vm $VM_NAME" + xe vm-export vm=$VM_UUID filename=$TMP_DIR/$VM_NAME.xva compress=$COMPRESS + ltest_state "export vm $VM_NAME" + + putlog "start vm $VM_NAME" + xe vm-start vm=$VM_UUID + ltest_state "run vm $VM_NAME" + sleep 60 + putlog "stop backup_vm_off $VM_NAME" +} + +backup_vm_off_snap(){ + VM_UUID=$1 + VM_NAME=$2 + COMPRESS=$3 + putlog "start backup_vm_off_snap $VM_NAME" + + putlog "shutdown vm $VM_NAME" + xe vm-shutdown vm=$VM_UUID + ltest_state "shutdown vm $VM_NAME" + sleep 60 + + putlog "create snapshot" + SNAP_UUID=$(xe vm-snapshot uuid=$VM_UUID new-name-label=snap-$VM_NAME) + ltest_state "create snapshot" + + putlog "start vm $VM_NAME" + xe vm-start vm=$VM_UUID + ltest_state "run vm $VM_NAME" + sleep 60 + + putlog "prepare export" + xe template-param-set is-a-template=false ha-always-run=false uuid=$SNAP_UUID + ltest_state "prepare export" + + putlog "export vm" + xe vm-export vm=$SNAP_UUID filename=$TMP_DIR/$VM_NAME.xva compress=$COMPRESS + ltest_state "export vm" + + putlog "delete snapshot" + xe vm-uninstall uuid=$SNAP_UUID force=true + ltest_state "delete snapshot" + + putlog "stop backup_vm_off_snap $VM_NAME" +} \ No newline at end of file