Changes between Version 8 and Version 9 of S3QLBackup


Ignore:
Timestamp:
Nov 3, 2015, 2:45:42 PM (3 years ago)
Author:
chris
Comment:

S3QL backup script updated, see ticket:39#comment:11

Legend:

Unmodified
Added
Removed
Modified

  • S3QLBackup

    v8 v9  
    1111{{{
    1212#!bash
    13 #!/bin/bash
     13in/bash
    1414
    1515# Abort entire script if any command fails
    16 set -e
     16#set -e
    1717
    1818# This script assumes that bucketname and directory under
    … …  
    2222CONFIG_DIR="/etc/s3ql"
    2323# S3 server URL
    24 SERVER="s3c://s.qstack.advania.com:443"
     24S3QL_SERVER="s3c://s.qstack.advania.com:443"
    2525# rsync command
    2626RSYNC="rsync -aHAXx --delete-during --delete-excluded --partial -v"
    … …  
    3131# directory where the scripts this script depends on are located
    3232SCRIPT_DIR="/usr/local/bin"
     33# This script uses lockfile which comes with procmail
     34# lockfile lines commented out, to enable see lines starting with ##
     35##LOCKFILE_BINARY="/usr/bin/lockfile"
     36##LOCKFILE="/var/run/lock/$(basename $0).lock"
     37#echo "LOCKFILE: $LOCKFILE"
     38# Seconds between unmount attempts
     39SLEEP="15"
     40# Mins after rsync before umnt it attempted
     41LONG_SLEEP="5m"
     42
     43# echo to standard out function
     44echoerr()
     45{
     46  echo "$@" 1>&2;
     47}
     48
     49# Check that the script is being run by root
     50if [[ "$(id -u)" != "0" ]] ; then
     51  echoerr "You must run '$0' as root or via sudo"
     52  exit 1
     53fi
     54
     55# Test if the lockfile binary can be found
     56##if [[ ! -e "$LOCKFILE_BINARY" ]]; then
     57##  echo "$LOCKFILE_BINARY not found, please install the procmail package."
     58##  exit 1
     59##fi
    3360
    3461# check the $S3QL_MOUNT directory exists
    3562if [[ ! -d "$S3QL_MOUNT" ]]; then
    36   echo "$S3QL_MOUNT doesn't exist"
     63  echoerr "$S3QL_MOUNT doesn't exist"
    3764  exit 1
    3865fi
    … …  
    4067# check the $SSHFS_MOUNT directory exists
    4168if [[ ! -d "$SSHFS_MOUNT" ]]; then
    42   echo "$SSHFS_MOUNT doesn't exist"
     69  echoerr "$SSHFS_MOUNT doesn't exist"
    4370  exit 1
    4471fi
    … …  
    4673# Check for bucket name / directory on standard input
    4774if [[ $1 ]]; then
    48   BUCKET=$1
    49 elif [[ ! $1 ]]; then
     75  BUCKET="$1"
     76elif [[ ! "$1" ]]; then
    5077  echo "Type the bucketname and then [ENTER]:"
    5178  read bucket
    52   BUCKET=$bucket
    53 fi
    54 
    55 # Check that a list of diectories to backup exists at
     79  BUCKET="$bucket"
     80fi
     81
     82# check if the Transport endpoint is not connected
     83if $SCRIPT_DIR/s3ql_endpoint_check ; then
     84  echo "No problem with disconnected endpoints"
     85else
     86  echoerr "Problem with a disconnected endpoint"
     87  exit 1
     88fi
     89
     90# Check that a list of directories to backup exists at
    5691# $CONFIG_DIR/$BUCKET
    5792if [[ ! -f "$CONFIG_DIR/$BUCKET" ]]; then
    58   echo "You need to create $CONFIG_DIR/$BUCKET with a list of directories to backup"
     93  echoerr "You need to create $CONFIG_DIR/$BUCKET with a list of directories to backup"
    5994  exit 1
    6095else
    … …  
    66101if [[ -d "$SSHFS_SOURCE" ]]; then
    67102  # mount the sshfs
    68   $SCRIPT_DIR/mnt-sshfs $BUCKET && \
    69   echo "Success mounting $SSHFS_MOUNT/$BUCKET" || \
    70 { echo "Problem mounting $SSHFS_SOURCE" ; exit 1 ; }
    71 else
    72   echo "$SSHFS_SOURCE doesn't exist"
     103  if $SCRIPT_DIR/mnt-sshfs "$BUCKET" ; then
     104    echo "Success mounting $SSHFS_MOUNT/$BUCKET"
     105  else
     106    echoerr "Problem mounting $SSHFS_SOURCE"
     107    exit 1
     108  fi
     109else
     110  echoerr "$SSHFS_SOURCE doesn't exist"
    73111  exit 1
    74112fi
    … …  
    77115if [[ -d "$S3QL_MOUNT/$BUCKET" ]]; then
    78116  # mount the s3ql directory
    79   $SCRIPT_DIR/mnt-s3ql $BUCKET  && \
    80   echo "Success mounting $S3QL_MOUNT/$BUCKET" || \
    81 { echo "Problem mounting $S3QL_MOUNT/$BUCKET" ; exit 1 ; }
    82 else
    83   echo "$S3QL_MOUNT/$BUCKET doesn't exist"
    84   exit 1
    85 fi
    86 
    87 # The following two commands are commented out as the mnt-s3ql script covers
    88 # this
    89 
    90 # Recover cache if e.g. system was shut down while fs was mounted
    91 #fsck.s3ql --backend-options="dumb-copy" --batch "$SERVER/$BUCKET"
    92 
    93 # Mount file system
    94 #mount.s3ql --backend-options="dumb-copy" "$SERVER/$BUCKET" "$S3QL_MOUNT/$BUCKET"
     117  if $SCRIPT_DIR/mnt-s3ql "$BUCKET" ; then
     118    echo "Success mounting $S3QL_MOUNT/$BUCKET"
     119  else
     120    echoerr "Problem mounting $S3QL_MOUNT/$BUCKET"
     121    exit 1
     122  fi
     123else
     124  echoerr "$S3QL_MOUNT/$BUCKET doesn't exist"
     125  exit 1
     126fi
     127
     128# if the $LOCKFILE exists then exit
     129# the lockfile is read only
     130# the timeout is set to 2 hours (7200 secs)
     131# if the lockfile is older than this it will be removed
     132##DATE=$(date -R)
     133#$LOCKFILE_BINARY -r 1 -l 21600 $LOCKFILE || { echo "$LOGFILE exists exiting $DATE" ; exit 23 ; }
     134##$LOCKFILE_BINARY -r 1 -l 7200 $LOCKFILE || \
     135##  { echo "$LOCKFILE exists exiting $DATE" ; exit 1 ; }
    95136
    96137# Figure out the most recent backup
    … …  
    107148NEW_BACKUP=`date "+%Y-%m-%d_%H:%M:%S"`
    108149if [[ -n "$LAST_BACKUP" ]]; then
    109     echo "Copying $LAST_BACKUP to $NEW_BACKUP..."
     150    echo "Copying ${LAST_BACKUP} to ${NEW_BACKUP}..."
    110151    s3qlcp "$LAST_BACKUP" "$NEW_BACKUP"
    111152
    112153    # Make the last backup immutable
    113154    # (in case the previous backup was interrupted prematurely)
     155    echo "Locking ${LAST_BACKUP}"
    114156    s3qllock "$LAST_BACKUP"
    115157fi
    116158
    117 # ..and update the copy
    118 #rsync -aHAXx --delete-during --delete-excluded --partial -v \
    119 #    --exclude /.cache/ \
    120 #    --exclude /.s3ql/ \
    121 #    --exclude /.thumbnails/ \
    122 #    --exclude /tmp/ \
    123 #    "/home/my_username/" "./$NEW_BACKUP/"
    124 
     159# Check directories exist and run the rsync
    125160if [[ ! -d "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP" ]]; then
    126161  echo "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP doesn't exist so creating it"
    127   mkdir -p "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP" || \
    128     { echo "Problem making $S3QL_MOUNT/$BUCKET/$NEW_BACKUP" ; exit 1 ; }
     162  if mkdir -p "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP" ; then
     163    echo "Created $S3QL_MOUNT/$BUCKET/$NEW_BACKUP"
     164  else
     165    echoerr "Problem making $S3QL_MOUNT/$BUCKET/$NEW_BACKUP"
     166    exit 1
     167  fi
    129168fi
    130169for dir in $(<${BACKUP_LIST}); do
    131170  mkdir -p "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP$dir/" || \
    132171    { echo "Problem making $S3QL_MOUNT/$BUCKET/$NEW_BACKUP$dir/" ; exit 1 ; }
    133   $RSYNC "$SSHFS_SOURCE$dir/" "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP$dir/" || \
    134     { echo "Problem with $RSYNC $SSHFS_SOURCE$dir/ $SSHFS_MOUNT/$BUCKET/$NEW_BACKUP$dir/" ; exit 1 ; }
     172  #$RSYNC "$SSHFS_SOURCE$dir/" "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP$dir/" || \
     173  #  { echo "Problem with $RSYNC $SSHFS_SOURCE$dir/ $SSHFS_MOUNT/$BUCKET/$NEW_BACKUP$dir/" ; rm -f "${LOCKFILE}" ; exit 1 ; }
     174  RSYNC_CMD="$RSYNC ${SSHFS_SOURCE}${dir}/ ${S3QL_MOUNT}/${BUCKET}/${NEW_BACKUP}${dir}/"
     175  if $RSYNC_CMD ; then
     176    echo "Success running rsync"
     177  else
     178    echo "Problem with rsync, running this script again"
     179    $0 $1
     180  fi
    135181done
    136182
    137183# Make the new backup immutable
     184echo "Locking ${NEW_BACKUP}"
    138185s3qllock "$NEW_BACKUP"
     186# cd to home so the s3ql file system can be unmounted
     187echo "Changing directory to $HOME so ${S3QL_MOUNT}/${BUCKET} can be unmounted"
     188cd "${HOME}"
     189# unmount s3ql first, but sleep first so background tasks can complete
     190echo "Sleeping for $LONG_SLEEP" ; sleep $LONG_SLEEP
     191if $SCRIPT_DIR/umnt-s3ql "${BUCKET}" ; then
     192  echo "Success unmounting ${S3QL_SERVER}/${BUCKET}"
     193else
     194  echo "Sleeping for ${LONG_SLEEP} and then using force"
     195  sleep ${LONG_SLEEP}
     196  if $SCRIPT_DIR/umnt-s3ql --force "${BUCKET}" ; then
     197    echo "Success using force unmounting ${S3QL_SERVER}/${BUCKET}"
     198  else
     199    echoerr "Failure using force to unmount ${S3QL_SERVER}/${BUCKET}"
     200    exit 1
     201  fi
     202fi
     203$SCRIPT_DIR/umnt-sshfs "${BUCKET}"
     204
     205# remove the lock file
     206##echo "Removing the lock at ${LOCKFILE}"
     207##rm -f "${LOCKFILE}"
    139208
    140209# Expire old backups
    … …  
    145214# be installed, and it *may* also not have the .py ending.
    146215#expire_backups --use-s3qlrm 1 7 14 31 90 180 360
    147 
    148 # unmount s3ql first
    149 #umnt-s3ql $BUCKET
    150 #umnt-sshfs $BUCKET
    151216}}}
    152217