S3QL Backup Script

Following is a modified version of the ​s3_backup.sh script which which was provided by the Debian package at /usr/share/doc/s3ql/examples/s3ql_backup.sh.

It expects a /etc/s3ql/SERVERNAME file listing directories to be backed up for each server and the SERVERNAME is expected to match the bucket name.

s3ql_backup

#!/bin/bash

# Abort entire script if any command fails
set -e

# This script assumes that bucketname and directory under 
# /media match and is also assumes that a file, with the 
# same name with a list of directories to backup exists 
# in /etc/s3ql
CONFIG_DIR="/etc/s3ql"
# S3 server URL
SERVER="s3c://s.qstack.advania.com:443"
# rsync command
RSYNC="rsync -aHAXx --delete-during --delete-excluded --partial -v"
# mount directory for s3ql file systems
S3QL_MOUNT="/media/s3ql"
# mount directory for sshfs file systems
SSHFS_MOUNT="/media/sshfs"
# directory where the scripts this script depends on are located
SCRIPT_DIR="/usr/local/bin"

# check the $S3QL_MOUNT directory exists
if [[ ! -d "$S3QL_MOUNT" ]]; then
  echo "$S3QL_MOUNT doesn't exist"
  exit 1
fi

# check the $SSHFS_MOUNT directory exists
if [[ ! -d "$SSHFS_MOUNT" ]]; then
  echo "$SSHFS_MOUNT doesn't exist"
  exit 1
fi

# Check for bucket name / directory on standard input
if [[ $1 ]]; then
  BUCKET=$1
elif [[ ! $1 ]]; then
  echo "Type the bucketname and then [ENTER]:"
  read bucket
  BUCKET=$bucket
fi

# Check that a list of diectories to backup exists at 
# $CONFIG_DIR/$BUCKET
if [[ ! -f "$CONFIG_DIR/$BUCKET" ]]; then
  echo "You need to create $CONFIG_DIR/$BUCKET with a list of directories to backup"
  exit 1
else
  BACKUP_LIST="$CONFIG_DIR/$BUCKET"
fi

# sshfs mounted at /media/server-name
SSHFS_SOURCE="$SSHFS_MOUNT/$BUCKET"
if [[ -d "$SSHFS_SOURCE" ]]; then
  # mount the sshfs
  $SCRIPT_DIR/mnt-sshfs $BUCKET && \
  echo "Success mounting $SSHFS_MOUNT/$BUCKET" || \
{ echo "Problem mounting $SSHFS_SOURCE" ; exit 1 ; }
else
  echo "$SSHFS_SOURCE doesn't exist"
  exit 1
fi

# check the bucket exists
if [[ -d "$S3QL_MOUNT/$BUCKET" ]]; then
  # mount the s3ql directory
  $SCRIPT_DIR/mnt-s3ql $BUCKET  && \
  echo "Success mounting $S3QL_MOUNT/$BUCKET" || \
{ echo "Problem mounting $S3QL_MOUNT/$BUCKET" ; exit 1 ; }
else
  echo "$S3QL_MOUNT/$BUCKET doesn't exist" 
  exit 1
fi

# The following two commands are commented out as the mnt-s3ql script covers
# this 

# Recover cache if e.g. system was shut down while fs was mounted
#fsck.s3ql --backend-options="dumb-copy" --batch "$SERVER/$BUCKET"

# Mount file system
#mount.s3ql --backend-options="dumb-copy" "$SERVER/$BUCKET" "$S3QL_MOUNT/$BUCKET"

# Figure out the most recent backup
cd "$S3QL_MOUNT/$BUCKET"
LAST_BACKUP=`python <<EOF
import os
import re
backups=sorted(x for x in os.listdir('.') if re.match(r'^[\\d-]{10}_[\\d:]{8}$', x))
if backups:
    print backups[-1]
EOF`

# Duplicate the most recent backup unless this is the first backup
NEW_BACKUP=`date "+%Y-%m-%d_%H:%M:%S"`
if [[ -n "$LAST_BACKUP" ]]; then
    echo "Copying $LAST_BACKUP to $NEW_BACKUP..."
    s3qlcp "$LAST_BACKUP" "$NEW_BACKUP"

    # Make the last backup immutable
    # (in case the previous backup was interrupted prematurely)
    s3qllock "$LAST_BACKUP"
fi

# ..and update the copy
#rsync -aHAXx --delete-during --delete-excluded --partial -v \
#    --exclude /.cache/ \
#    --exclude /.s3ql/ \
#    --exclude /.thumbnails/ \
#    --exclude /tmp/ \
#    "/home/my_username/" "./$NEW_BACKUP/"

if [[ ! -d "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP" ]]; then
  echo "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP doesn't exist so creating it"
  mkdir -p "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP" || \
    { echo "Problem making $S3QL_MOUNT/$BUCKET/$NEW_BACKUP" ; exit 1 ; }
fi
for dir in $(<${BACKUP_LIST}); do
  mkdir -p "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP$dir/" || \
    { echo "Problem making $S3QL_MOUNT/$BUCKET/$NEW_BACKUP$dir/" ; exit 1 ; }
  $RSYNC "$SSHFS_SOURCE$dir/" "$S3QL_MOUNT/$BUCKET/$NEW_BACKUP$dir/" || \
    { echo "Problem with $RSYNC $SSHFS_SOURCE$dir/ $SSHFS_MOUNT/$BUCKET/$NEW_BACKUP$dir/" ; exit 1 ; }
done

# Make the new backup immutable
s3qllock "$NEW_BACKUP"

# Expire old backups

# Note that expire_backups.py comes from contrib/ and is not installed
# by default when you install from the source tarball. If you have
# installed an S3QL package for your distribution, this script *may*
# be installed, and it *may* also not have the .py ending.
#expire_backups --use-s3qlrm 1 7 14 31 90 180 360

# unmount s3ql first
#umnt-s3ql $BUCKET
#umnt-sshfs $BUCKET