#! /bin/bash
# Possible exit codes:
# 0 - success
# 1 - permanent error (config - ie, backup will NEVER work with current config)
# 2 - warnings were issues but otherwise we had success.
# 3 - temporary error (ie, a re-run may succeed).

set -o errexit
set -o pipefail

# config
base=/mnt/backup
lvsnapprefix=bck
# allocation policies.
cf_snap_alloc_percentage=60 # Percentage of free space to use for creating snapshots.
cf_snap_max_util=65 # How full is a snap allowed to become
cf_snap_extend=15 # Percentage wise to extend (@ 15 % we can extend all snaps three times with 10% free space to spare)

dbbackuplockf=/tmp/.ulsdbbackup-shared

# parameters
dest=$1
conf=$2

# Prevent SSH using the forwarded socket.
unset SSH_AUTH_SOCK

# program paths/alterntives
awk=/usr/bin/gawk

# Make sure /bin and /usr/bin is in the path...
[[ ":${PATH}:" = *:/usr/bin:* ]] || PATH=$PATH:/usr/bin
[[ ":${PATH}:" = *:/bin:* ]] || PATH=$PATH:/bin

function log_exec() {
	local stime=$(date +%s)
	echo -n "${1}: "
	shift

	if outp=$("$@" 2>&1 | tail -n1000); then
		local etime=$(date +%s)
		echo "Ok (completed in $(( etime - stime)) seconds)."
		r=0
	else
		r=$?
		echo "Failed (excode=$r)."
		echo "Failed command:" "$@"
		cat <<HERE
${outp}
HERE
	fi
	return $r
}

function log_eval() {
	echo -n "${1}: "
	shift

	if outp=$(eval "$*" 2>&1); then
		echo "Ok."
		r=0
	else
		r=$?
		echo "Failed (excode=$r)."
		echo "Failed command:" "$@"
		cat <<HERE
${outp}
HERE
	fi
	return $r
}

function echoexec() {
	echo "$@"
	"$@"
}

function fail() {
	ex=$1; shift
	echo "$@" >&2
	exit $ex
}

function info() {
	echo "$@" >&2
	return 0
}

warnings=()
function warning() {
	echo "WARNING: $*"
	warnings=("${warnings[@]}" "$*")
	return 0
}

function filecontents() {
	[ -f "$1" ] && cat "$1" || /bin/true
}

mount_points=()
function do_mount() {
	log_exec "$1" /bin/mount "$2" "$3" || return $?
	mount_points=("${mount_points[@]}" "$3")
	return 0
}

function do_umount() {
	local mp

	if /usr/bin/lsof "$2" &>/dev/null; then
		info "Waiting for processes accessing $2 to terminate."
		while /usr/bin/lsof "$2" &>/dev/null; do
			sleep 0.5
		done
	fi
	log_exec "$1" /bin/umount "$2" || return $?
	local tmp=("${mount_points[@]}")
	mount_points=()
	for mp in "${tmp[@]}"; do
		[[ "${mp}" != "$2" ]] && mount_points=("${mount_points[@]}" "${mp}")
	done
	return 0
}

function cleanup() {
	local mp

	info "Cleaning up."

	for mp in "${mount_points[@]}"; do
		do_umount "Umount left-over mount point (${mp})" "${mp}"
	done

	if [ -n "${shadowdrive}" -a -d "/cygdrive/${shadowdrive}" ]; then
		shadowunexpose
	fi

	return 0
}

function calc_expiry_days() {
	local expiry_days=

	[ "$(type -t custom_calc_expiry)" == "function" ] && expiry_days=$(custom_calc_expiry "$1")
	[ -z "${expiry_days}" -a "$1" = "@0" ] && expiry_days=365
	[ -z "${expiry_days}" -a "$(date +%d -d "$1")" == "01" ] && expiry_days=365
	[ -z "${expiry_days}" -a "$(date +%u -d "$1")" == "5" ] && expiry_days=30
	[ -z "${expiry_days}" ] && expiry_days=7

	echo "${expiry_days}"
}

function calc_expiry() {
	local max_expiry_days
	local expiry_days
	local last_start
	local lastsuccess=$(getmeta lastsuccess)
	local bck_name="${backup_name}"
	local nowtime="$(date +%s)"

	max_expiry_days="$(calc_expiry_days "@${nowtime}")"
	if [ -n "${lastsuccess}" ]; then
		backup_name="backup-${lastsuccess}"
		last_start="$(date +%s -d "$(getbackupmeta started)")"

		echo "Last start: $(date -d@${last_start})" >&2
		(( last_start += 86400 ))
		while [ "${last_start}" -lt "${nowtime}" ]; do
			expiry_days=$(calc_expiry_days "@${last_start}")
			echo "Testing $(date -d "@${last_start}") => ${expiry_days}" >&2
			[ "${expiry_days}" -gt "${max_expiry_days}" ] && max_expiry_days="${expiry_days}"
			(( last_start += 86400 ))
		done

		backup_name="${bck_name}"
	else
		expiry_days=$(calc_expiry_days "@0")
		[ "${expiry_days}" -gt "${max_expiry_days}" ] && max_expiry_days="${expiry_days}"
	fi

	echo "Calculated ${max_expiry_days} day backup." >&2

	date +%Y-%m-%d -d "${max_expiry_days} days"
}

function has_tool() {
	which "$1" &>/dev/null
}

function need_tool() {
	local tool="$1"; shift
	has_tool "$tool" && return 0
	local msg="$*"
	fail 1 "${msg:-Required tool $tool missing}"
}

shadowdrive=
function shadowwrap() {
	local tfile=$(mktemp)
	while [ $# -gt 0 ]; do
		echo -e "$1\r" >> "${tfile}"
		shift
	done
	diskshadow /s "$(cygpath --windows "${tfile}")" | dos2unix
	local r=$?
	rm "${tfile}"
	return $r
}

function shadowexpose() {
	local shadowid="$(getbackupmeta "shadow_id_$1")"
	[ -z "${shadowid}" ] && fail 1 "Error locating shadow id for drive $1: (part of set $(getbackupmeta "shadowsetid"))."

	local shadowinfo="$(shadowwrap "list shadows id $shadowid")"
	local shadowexposed="$(sed -nre 's/.*Exposed locally as: (.):.*/\U\1/p' <<<"$shadowinfo")"

	if [ -n "${shadowexposed}" ]; then
		shadowdrive="${shadowexposed}"
		echo "Shadow copy ${shadowid} is already exposed on ${shadowdrive}:, not re-exposing."
		return
	fi
	echo "$shadowinfo"

	if [ -z "${shadowdrive}" ]; then
		if [ -n "${cf_shadowdrive}" ]; then
			shadowdrive="${cf_shadowdrive}"
		else
			for c in {Z..A}; do
				[ -d "/cygdrive/$c" ] && continue
				shadowdrive=$c
				break
			done
		fi
		[ -n "${shadowdrive}" ] || fail 1 "Unable to locate a suitable drive letter for exposing shadow copies on."
	fi

	[ ! -d "/cygdrive/${shadowdrive}" ] || fail 1 "Shadow drive ${shadowdrive}: is already in use."

	log_exec "Exposing shadow ${shadowid} for $1: on ${shadowdrive}: drive" shadowwrap "expose ${shadowid} ${shadowdrive}:"
}

function shadowunexpose() {
	[ -z "${shadowdrive}" ] && fail 1 "Unable to unexpose not used shadow drive."
	[ ! -d "/cygdrive/${shadowdrive}" ] || log_exec "Unexposing ${shadowdrive}: shadow copy" shadowwrap "unexpose ${shadowdrive}:"
	shadowdrive=
}

systemtype="$(uname -o)"

ssh_identity_file=
[ -z "${ssh_identity_file}" -a -r /etc/uls/backup.id_rsa ] && ssh_identity_file=/etc/uls/backup.id_rsa
[ -z "${ssh_identity_file}" -a -r /etc/uls/backup.id_dsa ] && ssh_identity_file=/etc/uls/backup.id_dsa
[ "${systemtype}" = Cygwin ] && ssh_identity_file="$(cygpath --windows "${ssh_identity_file}")"

[ -r /etc/uls/backup.conf ] && . /etc/uls/backup.conf
if [ -n "${conf}" ]; then
	if [ -r "/etc/uls/backup-${conf}.conf" ]; then
		. "/etc/uls/backup-${conf}.conf"
	else
		fail 1 "You requested config ${conf} be used, however /etc/uls/backup-${conf}.conf does not exist."
	fi
fi

#info "Using snapshot percentages init/maxutil/extend $cf_snap_alloc_percentage/$cf_snap_max_util/$cf_snap_extend."

## Functions used for the local fs implementation (or something that is 'mounted' locally at least)
function fs_getmeta() {
	filecontents "${metaloc}/.uls_$1"
}

function fs_setmeta() {
	attr=$1; shift
	echo "$@" > "${metaloc}/.uls_$attr"
}

function fs_getbackupmeta() {
	filecontents "${metaloc}/${backup_name}/.uls_$1"
}

function fs_setbackupmeta() {
	attr=$1; shift
	echo "$@" > "${metaloc}/${backup_name}/.uls_$attr"
}

function fs_mkbackup() {
	mkdir -p "${dstloc}/$1"
}

function fs_mkbackuppart() {
	mkdir -p "${dstloc}/${backup_name}/$1"
}

function fs_getbackuplist() {
	local d=
	for d in "${dstloc}"/*/.uls_started; do
		echo "$(basename "$(dirname "${d}")")"
	done
}

function fs_purge() {
	rm -rf "${dstloc}"/"${1}"
}

function fs_has_path_in_backup() {
	test -r "${dstloc}/backup-${1}/${2}/${3}"
}

## Functions used for backups done over ssh links.
ssh="ssh -q -o ServerAliveInterval=${cf_ssh_alive-30}"
[ -r "${ssh_identity_file}" ] && ssh+=" -i ${ssh_identity_file}"
function ssh_getmeta() {
	local fname="${sshbasedir}/.meta/$1"
	${ssh} "${sshserver}" -- "test -f ${fname} && cat ${fname} || /bin/true" || fail 3 "Error retrieving remote meta($1)."
}

function ssh_setmeta() {
	attr=$1; shift
	echo "$@" | ${ssh} "${sshserver}" "cat > ${sshbasedir}/.meta/${attr}" || fail 3 "Error setting remote meta($1)."
}

function ssh_getbackupmeta() {
	local fname="${sshbasedir}/${backup_name}/.$1"
	${ssh} "${sshserver}" -- "test -f ${fname} && cat ${fname} || /bin/true" || fail 3 "Error retrieving remote backup meta(${backup_name}:$1)."
}

function ssh_setbackupmeta() {
	attr=$1; shift
	echo "$@" | ${ssh} "${sshserver}" "cat > ${sshbasedir}/${backup_name}/.${attr}" || fail 3 "Error setting remote backup meta($1)."
}

function ssh_mkbackup() {
	${ssh} "${sshserver}" "mkdir -p ${sshbasedir}/$1" || fail 3 "Error creating base folder for backup ($1)."
}

function ssh_mkbackuppart() {
	${ssh} "${sshserver}" "mkdir -p ${sshbasedir}/${backup_name}/$1" || fail 3 "Error creating remote backup folder for partition ($1)."
}

function ssh_getbackuplist() {
	local d=
	for d in $(${ssh} "${sshserver}" "ls ${sshbasedir}/*/.started"); do
		echo "$(basename "$(dirname "${d}")")"
	done
}

function ssh_purge() {
	${ssh} "${sshserver}" "rm -rf '${sshbasedir}/${1}'"
}

function ssh_has_path_in_backup() {
	${ssh} "${sshserver}" "test -r '${sshbasedir}/backup-${1}/${2}/${3}'"
}

function fn_exists() {
	type "$1" 2>/dev/null | grep -q "^$1 is a function\$"
}

## Wrappers to hide the ugliness from stuff below.
function getmeta() { ${scheme}_getmeta "$@"; }
function setmeta() { ${scheme}_setmeta "$@"; }
function getbackupmeta() { ${scheme}_getbackupmeta "$@"; }
function setbackupmeta() { ${scheme}_setbackupmeta "$@"; }
function mkbackup() { ${scheme}_mkbackup "$@"; }
function mkbackuppart() { ${scheme}_mkbackuppart "$@"; }
function getbackuplist() { ${scheme}_getbackuplist; }
function purge() { ${scheme}_purge "$@"; }
function has_path_in_backup() { ${scheme}_has_path_in_backup "$@"; }

function rsync_retry() {
	local desc="$1"; shift
	local maxretry="${cf_rsync_retry:-3}"

	while true; do
		if log_exec "Running rsync for ${desc}" rsync "$@"; then
			break;
		else
			r=$?
			# 23 - Partial transfer due to error
			# 24 - Partial transfer due to vanished source files
			[ $r -eq 23 -o $r -eq 24 ] && fail 3 "Need to rerun backup due to failed LVM container."
			# Error codes as follows:
			# 12 - Error in rsync protocol data stream (connection reset by peer)
			# 30 - Timeout in data send/receive
			# 255 - unknown
			[ $r -eq 12 ] || [ $r -eq 30 ] || [ $r -eq 255 ] || fail 1 "non-recoverable rsync error."

			if [ $((--maxretry)) -eq 0 ]; then
				fail 3 "Too many failures, bailing out."
			else
				info "$maxretry retries left.  Retrying."
				if [ -n "${cf_rsync_retry_delay}" ]; then
					sleep "${cf_rsync_retry_delay}"
				fi
			fi
		fi
	done
}

info "Backup destination: ${dest}"

if [[ "${dest}" = /dev/sd[a-z] ]] || [[ "${dest}" = /dev/sd[a-z][0-9] ]] || [[ "$(readlink -f "${dest}")" = /dev/dm-* ]] || [[ "${dest}" = /dev/disk/* ]]; then
	if [ -n "${cf_waitblock}" ]; then
		mdelay=$(( $(date +%s) + ${cf_waitblock} ))
		while [ ! -b "${dest}" -a "$(date +%s)" -lt $mdelay ]; do
			sleep 1
		done
	fi
	[[ "${dest}" = /dev/disk/* ]] && dest=$(readlink -f "${dest}")
	if [[ "${dest}" = *[0-9] ]] || [[ "$(readlink -f "${dest}")" = /dev/dm-* ]]; then
		part="${dest}"
		[[ -e "${part}" ]] || fail 1 "Specified partition ${part} does not exist."
	else
		drive="${dest}"
		[[ -e "${drive}" ]] || fail 1 "Specified drive does not exist."
		part="${drive}1"
		[[ -e "${part}" ]] || fail 1 "Specified drive ${drive} does not contain a partition numbered 1"
	fi

	label="$(/sbin/blkid "${part}" -s LABEL | ${awk} -F'"' '{ print $2 }')"
	[ -z "${label}" ] && fail 1 "Disk partition does not contain a volume label required for managing backups."

	base="${base}/${label}"
	dstloc="${base}/dst"

	mkdir -p "${dstloc}" || fail 1 "Error creating mountpoint"
	do_mount "Mounting destination drive" "${part}" "${dstloc}" || fail 1 "Error mounting backup drive."

	metaloc="${dstloc}"
	scheme=fs
elif [[ "${dest}" = /* ]] && [ -d "${dest}" ]; then
	label="$(basename "${dest}")"
	base="${base}/${label}"
	dstloc="${dest}"
	metaloc="${dstloc}"
	scheme=fs
elif [[ "${dest}" = ssh://* ]]; then
	sshserver="$(echo "${dest}" | sed -re 's,ssh://(([^@]*@)?[^:@]*):.*,\1,')"
	sshbasedir="$(echo "${dest}" | sed -re 's,ssh://([^@]*@)?[^:@]*:(.*),\2,')"
	dstloc="${dest##ssh://}"
	scheme=ssh
	label="${sshserver##*@}"
	base="${base}/${label}"
else
	fail 1 "Unknown destination schema for backup destination '${dest}'"
fi

function find_mountpoint()
{
	local mp="$1"
	while ! mountpoint -q "${mp}"; do
		mp="${mp%/*}"
		[ -z "${mp}" ] && mp=/
	done
	echo "${mp}"
}

function in_array()
{
	local needle="$1"
	shift
	while [ $# -gt 0 ]; do
		[ "$1" = "$needle" ] && return 0
		shift
	done
	return 1
}

##
# Used to close all file descriptors, except those numbers passed (which
# implicitly includes 0 1 and 2.
function close_fds() {
	local exemptlist=(0 1 2 "$@")
	local p
	for p in /proc/self/fd/*; do
		p="$(basename "$p")"
		in_array "$p" "${exemptlist[@]}" || eval "exec $p<&-"
	done
}

##
# Used to execute lvm related commands in a serialized manner (because the
# locking from lvm itself seems to be hap-hazzard).
#
# If config allows it (don't unless you know you need this), it'll attempt
# to force the command to succeed by clearing the udev cookie.
function lvop() {
	[ -x /sbin/lvs ] || return 0
	exec 4>/tmp/.ulsbackup.lvoplock
	flock -x 4
	(
		close_fds
		exec "$@"
	)
	local r=$?
	exec 4<&-
	return $r
}

function mysql_makesnaps() {
	local databases
	local db
	local innodbonly
	local locktype
	local nowarnvar
	local dbcompress
	local dbexten=sql
	[ -n "${cf_dbbackup_compress}" ] && dbcompress="| pbzip2 -9" && dbexten=".sql.bz2" && info "Compressing database dumps."

	# Only do this if mysql is running
	/etc/init.d/mysql --quiet status || return 0

	databases=($(MYSQL_PWD="${cf_dbbackup_pass}" mysql -u"${cf_dbbackup_user}" -B -N -e "show databases"))
	for db in "${databases[@]}"; do
		[ "${db}" = "information_schema" ] && continue
		[ "${db}" = "performance_schema" ] && continue
		[ "${db}" = "sys" ] && continue
		[ "${db}" = "#mysql50#lost+found" ] && continue
		[ "${db}" = "lost+found" ] && continue
		[ -x /usr/sbin/dbbackup -a "${db}" = "uls" -a -z "${cf_dbbackup_structonly}" ] && continue # we use dbbackup for this one (if we want a full dump).
		# Determine if we've got InnoDB only ...
		readarray -t noninnodbtables < <(MYSQL_PWD="${cf_dbbackup_pass}" mysql -u "${cf_dbbackup_user}" -B -N information_schema -e "SELECT table_name FROM tables WHERE engine NOT IN ('MEMORY','InnoDB') AND table_schema='${db}'")
		if [ "${#noninnodbtables[@]}" -eq 0 ]; then
			locktype="--single-transaction"
		else
			locktype="--lock-tables"
			nowarnvar="cf_dbnowarn_${db//[-]/_}"
			[ -n "${cf_dbnowarn}" ] && eval "${nowarnvar}=\"\${cf_dbnowarn}\""
			[ "${db}" != "mysql" -a -z "${!nowarnvar}" ] && warning "Database $db uses ${#noninnodbtables[@]} non-InnoDB tables$([ "${#noninnodbtables[@]}" -le 5 ] && echo " namely $(printf "%s, " "${noninnodbtables[@]}" | sed -re 's/, ([^,]+), $/ and \1/' -e 's/, $//')")."
		fi

		log_eval "Dumping database $db" "MYSQL_PWD='${cf_dbbackup_pass}' mysqldump -f -u'${cf_dbbackup_user}' -R ${locktype} --quick "${db}" ${cf_dbbackup_structonly:+--no-data} ${dbcompress} > /var/dbbackup/${db}.${dbexten}"
	done

	find /var/dbbackup -name "*.${dbexten}" -mtime +5 -exec rm {} +
	return 0
}

function mysql_flush() {
	local status

	# Only do this if mysql is running (and we have the tools available).
	/etc/init.d/mysql --quiet status || return 0
	[ -x /usr/sbin/myflush ] || return 0

	echo -n "Flushing MySQL tables to disk: "
	touch /tmp/.myflush
	exec 5< <(while [ -r /tmp/.myflush ]; do sleep 0.1; done)
	exec 6< <(/usr/sbin/myflush -u "${cf_dbbackup_user}" -p "${cf_dbbackup_pass}" <&5)
	read status <&6
	echo "$status"
	return 0
}

function mysql_release() {
	[ -r /tmp/.myflush ] || return 0
	echo "Releasing MySQL read lock."
	rm /tmp/.myflush
	exec 5<&-
	exec 6<&-
	return 0
}

function doinit() {
	excode=0
	"$@" || excode=$?
	[ $excode -eq 0 ] && return 0
	log_exec "Purging ${backup_name}" purge ${backup_name}
	log_exec "Resetting current backup to none since $1 failed" setmeta "current" ""
	fail $excode "Unable to execute init function: $*."
}

# Grab a lock to ensure only one backup instance is running per 'label'
if [ -x /usr/bin/flock ]; then
	flockname="/tmp/.uls-lock.ulsbackup.${label}.lock"
	exec 3>"${flockname}"
	if ! flock -x -w 5 3; then
		cat ${flockname}
		fail 3 "Error obtaining lock on ${flockname} - is another instance running?"
	fi
fi

total_stime=$(date +%s)

dstcfg=$(getmeta config)
[ -n "${dstcfg}" ] && eval "${dstcfg}"

trap cleanup EXIT

[ -d "${base}" ] || log_exec "Creating base (${base}) working folder" mkdir -p "${base}"

ourhost="${cf_hostname:-$(hostname)}"
host=$(getmeta host)
[[ "${host}" == "${ourhost}" ]] || fail 1 "Wrong backup location.  location is for '${host}', not '${ourhost}'."

last_success=$(getmeta lastsuccess)
current=$(getmeta current)

lvbckprefix="${lvsnapprefix}_${label}"

if [[ "${current}" == "$last_success" ]] && [[ -n "${current}" ]]; then
	current=
	setmeta current ""
fi

[ -n "${cf_backup_start}" ] && eval "${cf_backup_start}" || /bin/true

if [ -n "${current}" ]; then
	info "Resuming backup ${current} to ${label} at $(date "+%T, %Y-%m-%d")"
	backup_name="backup-${current}"
fi

if [ -z "${current}" ]; then
	if [ $(uname -o) = "GNU/Linux" ] && [ -x /sbin/lvs ]; then
		for i in $(lvop /sbin/lvs --noheadings | ${awk} '$1 ~ "'"${lvbckprefix}"'_" { print $2 "/" $1 }'); do
			log_exec "Removing old backup LVM snapshot $i" lvop /sbin/lvremove -An -ff "/dev/$i"
		done
	fi
	current=$(( last_success + 1 ))
	info "Initiating backup $current for ${ourhost} to $label at $(date "+%T, %Y-%m-%d")"
	backup_name="backup-${current}"
	mkbackup "${backup_name}"
	setbackupmeta "started" "$(date "+%Y-%m-%d %T")"
fi

if [ -z "$(getbackupmeta "expiry")" ]; then
	backup_expiry="$(calc_expiry)"
	info "Setting expiry to ${backup_expiry}"
	setbackupmeta "expiry" "${backup_expiry}"
fi

localstatus=$(getbackupmeta status)
if [ -z "${localstatus}" ]; then
	if [ ${#cf_backuppoints[@]} -eq 0 ]; then
		case "${systemtype}" in
			GNU/Linux)
				cf_backuppoints=(
					$(
						(
						/bin/df -Pl | while read a b; do echo "$(readlink -f "$a") $b"; done | \
							${awk} '$1 ~ "^/dev/" && $1 !~ "^/dev/mapper/" && $1 !~ "^/dev/dm-" && $NF !~ "^/mnt/" { print $NF }';
							lvop /sbin/lvs --noheadings | ${awk} '$3 !~ "^[st]" { print $2 "/" $1 }' | while read lvp; do [ "$(/sbin/blkid -o export /dev/$lvp | sed -nre 's/^TYPE=//p')" = swap ] || echo $lvp; done
						) | grep -v -f <(for skp in "${cf_backuppoints_skip[@]}"; do echo "^$skp$"; done)
					)
				)
			;;
			Cygwin)
				need_tool dos2unix
				need_tool diskshadow
				need_tool wmic
				need_tool rsync
				cf_backuppoints=($(wmic logicaldisk where drivetype=3 get DeviceID | sed -nre 's/^([A-Z]):[[:space:]]*$/shadow_\1/p'))
			;;
			*)
				fail 1 "Mount-point auto-detection not supported on ${systemtype}"
			;;
		esac
	fi
	info "Using backup points:"
	for bp in "${cf_backuppoints[@]}"; do info "- ${bp}"; done
	localstatus="$(for bp in "${cf_backuppoints[@]}"; do echo -e "${bp}\tnone"; done)"
	setbackupmeta "status" "${localstatus}"
	setmeta current $current

	if [ -n "${cf_dbbackup_user}" -a -n "${cf_dbbackup_pass}" ]; then
		doinit mysql_makesnaps
		if [ -x /usr/sbin/dbbackup -a -z "${cf_dbbackup_structonly}" ] && /etc/init.d/mysql --quiet status; then
			info "Running /usr/sbin/dbbackup --ulsbackup "${label}" --skipremote --nodbcompress"
			doinit /usr/sbin/dbbackup --ulsbackup "${label}" --skipremote --nodbcompress
		fi
	fi
	fn_exists bck_init && doinit bck_init $current $backup_name
fi

backup_root="${dstloc}/${backup_name}"

# Make snapshots of everything (that still needs to be backed up) ...
if [ "${systemtype}" = "GNU/Linux" ]; then
	[ -z "${flockname}" ] && fail 1 "flock locking is required when running on a Linux system."

	if [ -x /sbin/lvs ]; then
		lvop /sbin/lvs | awk '$3 ~ "^[Ss]...I" { print $2 "/" $1 }' | while read I; do
			log_exec "Removing corrupt snapshot $I" lvop /sbin/lvremove -An -ff "/dev/$I"
		done

		log_exec "Backing up VG metadata" lvop /sbin/vgcfgbackup
		info "Calculating snapshot ratio's (free/used)"
		eval "$(lvop /sbin/vgs --noheadings --units m --nosuffix | awk '{ print "ratio_" $1 "=" int( $NF / ( $(NF-1) - $NF) * 1000 ); print "info \" - "$1": $(( ${ratio_"$1"} / 1000 )).$(printf \"%03d\" $(( ${ratio_"$1"} % 1000 )))\""; }')"

		if [ -x /etc/init.d/dmeventd ] && /etc/init.d/dmeventd --quiet status; then
			info "Relying on dmeventd to auto-extend snapshots."
		else
			if pidof dmeventd &>/dev/null; then
				log_exec "Removing monitoring on existing snapshots" lvop /sbin/vgchange --monitor=n
				log_exec "Shutting down dmeventd" killall dmeventd
			fi

			lvm_monitoring="$(sed -nre 's/^[[:space:]]*monitoring[[:space:]]*=[[:space:]]*([^[:space:]]*)[[:space:]]*(#.*)?$/\1/p' /etc/lvm/lvm.conf)"
			if [ "${lvm_monitoring}" != "0" ]; then
				log_exec "Setting monitoring=0 in lvm.conf" sed -re 's/^([[:space:]]*monitoring[[:space:]]*=[[:space:]]*)([^[:space:]]*)([[:space:]]*)(#.*)?$/\10\3\4/' -i /etc/lvm/lvm.conf
			fi

			info "Starting lvs monitoring process."
			(
				close_fds
				exec 3>"${flockname}"
				while ! flock -x -w 1 3; do
					# $1 - LV
					# $2 - VG
					# $4 - snapshot size (MB)
					# $6 - snap %.
					extend_lvs=($(lvop /sbin/lvs --noheadings --units m 2>/dev/null | \
						awk '$3 ~ "^s" && $3 !~ "I" && $NF>='${cf_snap_max_util}' { print "/dev/" $2 "/" $1 }'))
					for lv in "${extend_lvs[@]}"; do
						skipvar="${lv//[.\/-]/_}"
						[ -z "${!skipvar}" ] || continue
						eval "$(lvop /sbin/lvs --units m --nosuffix --noheadings ${lv} | awk '{ print "snapsize="$4 * 100 "; origin=" $5 }')"
						originsize=$(lvop /sbin/lvs --units m --nosuffix --noheadings "${lv%/*}/${origin}" | awk '{ print $4 * 100 }')
						if [ "${originsize}" -lt "${snapsize}" ]; then
							eval "${skipvar}=1"
							continue
						fi

						lvop /sbin/lvextend -An -l+"${cf_snap_extend}"%LV "${lv}" &>/dev/null
					done
				done
			) &
		fi
	fi

	# If we have a mysql instance running, then we need to figure out where it's base mount
	# point sits, and flush with read lock it before we snapshot that LV.
	my_dev=-
	if [ -d /var/lib/mysql ]; then
		my_mountpoint="$(find_mountpoint /var/lib/mysql)"
		my_dev=$(mount | awk '$3=="'${my_mountpoint}'" { print $1 }')
		[ -n "${my_dev}" ] && my_dev=$(stat -c"%t.%T" -L "${my_dev}")
	fi

	# If we're going to invoke dbbackup, we should grab the shared lock.
	dbbackup_dev=-
	if [ -x /usr/sbin/dbbackup ] && [ -x /etc/init.d/mysql ] && /etc/init.d/mysql --quiet status; then
		dbbackup_mountpoint="$(find_mountpoint /var/dbbackup)"
		dbbackup_dev=$(mount | awk '$3=="'${dbbackup_mountpoint}'" { print $1 }')
		[ -n "${dbbackup_dev}" ] && dbbackup_dev=$(stat -c"%t.%T" -L "${dbbackup_dev}")
	fi

	echo "${localstatus}" | while read part state rest; do
		[[ $part = /* ]] && echo "Can't snapshot $part - not LVM" && continue # Ignore non-lvm sources
		[ "${state}" == "done" ] && info "Skipping snapshot for completed volume ${part}" && continue

		rdev="$(stat -c"%t.%T" -L /dev/${part})"
		[ "${my_dev}" = "${rdev}" ] && is_mysql=1 || is_mysql=
		[ "${dbbackup_dev}" = "${rdev}" ] && is_dbbackup=1 || is_dbbackup=

		if [ "${is_dbbackup}" ]; then
			[ -z "${cf_debugsharedlock}" ] || logger -t "dbbackup" "grabbing shared lock from ulsbackup"
			info "Grabbing dbbackup shared lock."
			exec 7>"${dbbackuplockf}"
			flock -s 7 || fail 2 "Unable to grab shared dbbackup lock."
		fi

		vg=${part%/*}
		lv=${part#*/}
		[ -b "/dev/${vg}/${lvbckprefix}_${lv}" ] && info "Re-using existing snapshot for ${part}" && continue
		errc=0
		ratio_var="ratio_${vg}"
		lvoriginsize="$(lvop /sbin/lvs "/dev/${part}" --units m --noheadings --nosuffix | awk '$3!~"^V" { print $4 } $3~"^V" { print "-" }')"
		[ -z "${lvoriginsize}" ] && fail 3 "Unable to determine origin size for ${part}"
		if [ "${lvoriginsize}" = "-" ]; then
			# We're on a "thin pool"
			unset lvsnapsize
		else
			lvsnapsize="$(( ${lvoriginsize/./} * ${cf_snap_alloc_percentage} * ${!ratio_var} / 40000000  * 4 ))"
			[ "${lvsnapsize}" -lt "${lvoriginsize%.*}" ] || lvsnapsize="${lvoriginsize%.*}"
			[ "${lvsnapsize}" -le 0 ] && lvsnapsize=1
		fi

		[ "${is_mysql}" ] && mysql_flush
		while ! log_exec "Creating snapshot for ${part} (${lvsnapsize-thin pool}${lvsnapsize+/${lvoriginsize} MB})" lvop /sbin/lvcreate ${lvsnapsize+-L}${lvsnapsize}${lvsnapsize+M} -kn -An -s -n ${lvbckprefix}_${lv} /dev/${part}; do
			[ -b "/dev/${vg}/${lvbckprefix}_${lv}" ] && log_exec "Removing failed snapshot" lvop /sbin/lvremove -An -ff /dev/${vg}/${lvbckprefix}_${lv}
			[ $(( ++errc )) -ge 5 ] && fail 3 "Too many failures attempting to create snapshot for ${part}"
		done
		[ "${is_mysql}" ] && mysql_release || /bin/true
	done
elif [ "${systemtype}" = Cygwin ]; then
	shadow_set_id=$(getbackupmeta shadowsetid)
	shadow_set_drives=$([ -z "${shadow_set_id}" ] || shadowwrap "list shadows set ${shadow_set_id}" | sed -nre 's/.*Original volume name: .* [[]([A-Z]):\\]/\1/p' | sort)

	shadow_desired_set="$(echo "${localstatus}" | while read part state rest; do
			[[ $part != shadow_* ]] || echo "${part#shadow_}"
		done | sort)"

	if [ "${shadow_desired_set}" = "${shadow_set_drives}" ]; then
		info "Re-using shadow set ${shadow_set_id}"
	else
		if [ -n "${shadow_set_id}" ]; then
			if [ -n "${shadowdrive}" -a -d "/cygdrive/${shadowdrive}" ]; then
				shadowunexpose
			fi
			log_exec "Nuking incorrect shadow set ${shadow_set_id}" shadowwrap "delete shadows set ${shadow_set_id}"
		fi

		shadow_args=( "set context persistent nowriters" "begin backup" )
		for d in ${shadow_desired_set}; do
			shadow_args+=("add volume $d: alias ULSB_$d")
		done
		shadow_args+=("create" "end backup")

		eval "$(shadowwrap "${shadow_args[@]}" | sed -nre 's/.*([{][-0-9a-f]+[}]).*%ULSB_([A-Z])%.*/shadow_drive_id_\2="\1"/p' -e 's/.*([{][-0-9a-f]+[}]).*%VSS_SHADOW_SET%.*/shadow_set_id="\1"/p')"

		setbackupmeta shadowsetid "${shadow_set_id}"
		for d in ${shadow_desired_set}; do
			vn="shadow_drive_id_${d}"
			setbackupmeta shadow_id_${d} "${!vn}"
		done
	fi
else
	fail 1 "Unrecognized system ${systemtype}"
fi

! which ionice &>/dev/null || ionice -c Best-Effort -n 7 -p $$

while true; do
	part=$(echo "${localstatus}" | ${awk} 'BEGIN {
		busy = ""
		none = ""
	}

	{
		if ($2 == "busy" && busy == "")
			busy = $1
		if ($2 == "none" && none == "")
			none = $1
	}

	END {
		if (busy != "")
			print busy
		else if (none != "")
			print none
	}')

	[[ -z "${part}" ]] && break

	state=$(echo "$localstatus" | ${awk} "{ if (\$1 == \"${part}\") print \$2 }")
	if [[ "$state" == "none" ]]; then
		info "Starting backup for ${part}"
		localstatus=$(echo "${localstatus}" | sed -e "s:^${part}\tnone$:${part}\tbusy:")
		setbackupmeta status "${localstatus}"
	else
		info "Resuming backup for ${part}"
	fi

	rsync_params=(
		"--rsh"
		"${ssh}"
	#	"--verbose"
	#	"--progress"

	#	"--archive"
		"--recursive"
		"--links"
		"--hard-links"
		"--perms"
		"--times"
		"--devices"
		"--specials"

		"--human-readable"
	#	"--itemize-changes"
		"--sparse"
		"--stats"

		"--numeric-ids"
		"--timeout=${cf_timeout-0}"
	)

	if [ "$(uname -o)" != "Cygwin" ]; then
		rsync_params=(
			"${rsync_params[@]}"
			"--group"
			"--owner"
		)
	fi

	[ -n "${cf_no_incremental}" ] && rsync_params=("${rsync_params[@]}" "--no-inc-recursive")

	if [ "${scheme}" == "ssh" ]; then
		rsync_params=("${rsync_params[@]}" "--compress")
	fi

	fldrname=${part//\//_}
	var_string=${fldrname//[^a-zA-Z0-9_]/_}

	arname="cf_exclude_${var_string}[@]"
	for exclude in "${!arname}"; do
		rsync_params=("${rsync_params[@]}" "--exclude" "${exclude}")
	done

	arname="cf_include_${var_string}[@]"
	for include in "${!arname}"; do
		rsync_params=("${rsync_params[@]}" "--include" "${include}")
	done

	dstfldr="${backup_root}/${var_string}"

	log_exec "Creating backup folder for ${part}" mkbackuppart "${fldrname}"

	if [[ "${part}" = [^/]* ]]; then
		if [[ "${systemtype}" = Cygwin ]]; then
			if [[ "${part}" = shadow_* ]]; then
				shadowexpose "${part#shadow_}"
				srcloc="/cygdrive/${shadowdrive}"
			else
				fail 1 "Don't know how to handle backup partition ${part}."
			fi
		else
			vg=${part%/*}
			lv=${part#*/}
			srcloc="${base}/${part}"
			mkdir -p "${srcloc}" || fail 3 "Unable to create snapshot mount point"
			do_mount "Mounting snapshot for ${part}" "/dev/${vg}/${lvbckprefix}_${lv}" "${srcloc}" || fail 3

			inodec="$(df -i "${srcloc}" -P | awk '$3~"^[0-9]+$" { print $3 }')"
			maxalloc=$(( ( inodec + 1023 ) / 1024 * 32 * 1024 )) # 32 * 1024^2 (~32m) files maps to 1G, round up to 32MB.
			if [ $maxalloc -gt $(( 1024 ** 3 )) ]; then
				rsync_params+=("--max-alloc=${maxalloc}")
			fi

			if [ -n "${cf_use_inode_hack}" ]; then
				log_exec "Preparing inode-based hard links for pre-sync ${srcloc}" $(dirname "$0")/ulsbackup_prep_inode_links "${srcloc}" "${!arname}"

				[ -z "${cf_no_incremental}" ] && rsync_params=("${rsync_params[@]}" "--no-inc-recursive")

				if [[ -n "${last_success}" ]]; then
					if has_path_in_backup "${last_success}" "${fldrname}" ".inode"; then
						rsync_retry "${part} (inode-sync)" "${rsync_params[@]}" --existing --link-dest "${dstloc#*:}/backup-${last_success}/${fldrname}/.inode/" "${srcloc}/.inode/" "${dstfldr}/.inode/"
					else
						info "Skipping inode-sync for ${part} since previous backup doesn't have .inode folder present."
						rsync_resync_params=("${rsync_params[@]}")
						rsync_params=("${rsync_params[@]}" "--exclude" "/.inode/")
					fi
				fi
			fi
		fi
	elif [[ "${part}" = /* ]]; then
		srcloc="${part}"
		rsync_params=(
			"${rsync_params[@]}"
			"--one-file-system"
		)
		info "Synching directly from ${part}"
	else
		fail 1 "Encountered unsupported volume specification: ${part}."
	fi

	# We may want to drop this if we are using the inode mechanism.
	if [[ -n "${last_success}" ]]; then
		info "Found previous backup for ${part}, using for reference"
		rsync_params=(
			"${rsync_params[@]}"
			"--fuzzy"
		)
		if has_path_in_backup "${last_success}" "${fldrname}" "."; then
			rsync_params=(
				"${rsync_params[@]}"
				"--link-dest" "${dstloc#*:}/backup-${last_success}/${fldrname}/"
			)
		fi
	fi

	rsync_retry "${part}" "${rsync_params[@]}" "${srcloc}/" "${dstfldr}/"

	if [ -n "${cf_use_inode_hack}" -a -n "${last_success}" -a -d "${srcloc}/.inode" -a -n "${rsync_resync_params+gotcha}" ]; then
		rsync_retry "${part} (initial inode-sync)" "${rsync_resync_params[@]}" "${srcloc}/" "${dstfldr}/"
	fi

	localstatus="$(echo "${localstatus}" | sed -re "s,^${part}\t(none|busy)$,${part}\tdone\t$(date "+%Y-%m-%d %T"),")"
	log_exec "Marking ${part} as done" setbackupmeta "status" "${localstatus}"

	if [[ "${part}" = [^/]* ]]; then
		if [ "${systemtype}" = Cygwin ]; then
			shadowunexpose
		else
			do_umount "Unmounting snapshot for ${part}" "${srcloc}"
			openc=$(lvop /sbin/lvdisplay "/dev/${vg}/${lvbckprefix}_${lv}" | sed -nre 's/^\s*# open\s+//p')
			if [ "${openc}" -gt 0 ]; then
				info "Waiting for the kernel to actually unmount the LV"
				while [ "${openc}" -gt 0 ]; do
					sleep 0.1
					openc=$(lvop /sbin/lvdisplay "/dev/${vg}/${lvbckprefix}_${lv}" | sed -nre 's/^\s*# open\s+//p')
				done
			fi
			log_exec "Destroying snapshot for ${part}" lvop /sbin/lvremove -An -ff "/dev/${vg}/${lvbckprefix}_${lv}"

			if [ "$(readlink -f "/dev/${part}")" = "${dbbackup_dev}" ]; then
				[ -z "${cf_debugsharedlock}" ] || logger -t "dbbackup" "releasing shared lock from ulsbackup"
				info "Releasing dbbackup shared lock."
				exec 7<&-
			fi
		fi
	elif [[ "${part}" = /* ]]; then
		info "Not unmounting ${part} (live system)"
	fi
done

if [ "${systemtype}" = Cygwin -a -n "${shadow_set_id}" ]; then
	log_exec "Destroying shadow set" shadowwrap "delete shadows set ${shadow_set_id}"
fi

log_exec "Marking backup ${current} as complete" setbackupmeta "completed" "$(date "+%Y-%m-%d %T")"
if [ "${systemtype}" = "GNU/Linux" ]; then
	destmd5=$(md5sum <<<"${dest}")
	rm -f "/root/.ulsbackup/${destmd5%% *}"
fi

log_exec "Marking backup ${current} as lastsuccess" setmeta "lastsuccess" "$current"
log_exec "Updating current backup to none" setmeta "current" ""

total_etime=$(date +%s)
total_ttime=$(( total_etime - total_stime ))
ttime=""
if [ "${total_ttime}" -ge 3600 ]; then
	ttime="$(( total_ttime / 3600 ))h"
fi
ttime+=$(printf "%02dm%02d" $(( total_ttime / 60 % 60 )) $(( total_ttime % 60 )))

info "Backup completed ($(date "+%Y-%m-%d %T"), completed in $ttime)."

today=$(date +%Y-%m-%d)
lastsuccess=$(getmeta lastsuccess)
minrotations=$(getmeta minrotations)
[ -z "${minrotations}" ] && minrotations=${cf_minrotations-7}

info "Checking for expired backups (${today}):"

for backup_name in $(getbackuplist); do
	exp=$(getbackupmeta expiry)
	[ -z "${exp}" ] && exp="0000-00-00"
	[[ "${exp}" < "${today}" ]] && action=purge || action=keep
	if [ "${action}" == "purge" ] && [ $(( $lastsuccess - ${backup_name##*-} )) -lt ${minrotations} ]; then
		action=keep
	fi
	info "=) ${backup_name} => ${exp} ($action)"
	! [ ${action} = "purge" ] || log_exec "Purging ${backup_name}" purge ${backup_name} || warning "Purge failed."
done

excode=$?

if [ "${#warnings[@]}" -gt 0 ]; then
	info "The following warnings were issued whilst performing the backup:"
	for w in "${warnings[@]}"; do
		info "* $w"
	done
	[ $excode -eq 0 ] && excode=4
fi

[ -n "${cf_backup_stop}" ] && eval "${cf_backup_stop} $excode" || /bin/true

exit $excode