bashclub-zfs-push-pull/bashclub-zfs

#!/bin/bash
# backup-zfs: use zfs send/recv to push/pull snapshots - New does not run twice
prog="$(basename "$0")"

usage() {
	cat >&2 <<-EOF
	usage: $prog [-hvq] [-t tag] [-k keep] [-d dateopts] src dest
	  use zfs send/recv to push/pull snapshots

	  !This Release is default set to send raw. This means you can send encrypted and compressed Datasets 1:1!

	  src          the source fs, specified as [host:]pool/path/to/fs
	  dest         the destination fs parent, specified as [host:]pool/path/to/fs
	               (the final path component of src will be appended to dest)
	  -p           ssh port
	  -h           help
	  -v           verbose mode
	  -C           SSH Compression - obsolete if send_opts="-v -w" are set to raw
	  -R           ZFS Re-Init
	  -I           Send intermediate Snapshots
	  -q           quiet mode
	  -t tag       tag to use for naming snapshots (default: backup-zfs)
	  -k keep      number of snapshots to keep on src (default: 5)
	  -d dateopts  options for date(1) - used to name the snapshots (default: +%F_%T)
	  -s           store mode - output snaps from local fs to ssh server
	  -r           read mode - read snaps from ssh server to local fs
	  -g gpg-id    gpg recipient key id (store mode only)
	EOF
	exit $1
}

# log to syslog; if verbose or on a tty, also to stdout
# usage: log msg
log() {
	logger -t "$prog" -- "$@"
	if ! $quiet && [[ -t 1 ]] || $verbose ; then
		echo "$@" >&2
	fi
}

# exit with a code & message
# usage: die $exitcode msg
die() {
	code="$1"
	shift
	if [[ $code -ne 0 ]] ; then
		verbose=true log "FATAL: $@"
	else
		log "$@"
	fi
	exit $code
}

# run zfs(1) command either locally or via ssh
# usage: ZFS "$host" command args...
ZFS() {
	host="$1"
	shift
	if [[ -n $host ]] ; then
		log "remote ($host): zfs $@"
		ssh $sshcompress -p $port "$host" zfs "$@"
	else
		log "local: zfs $@"
		zfs "$@"
	fi
}

(
    flock -n 9 || exit 1 
    ###
    ### defaults
    ###
    tag="$prog"
    dateopts="+%F_%T"
    keep=5
    verbose=false
    quiet=false
    tossh=false
    fromssh=false
    port=22
    sshcompress=""
    reinit=""
    intermediate="-i "
    ###
    ### parse options
    ###
    while getopts "hvqk:p:t:d:srCIRg:" opt ; do
        case $opt in
            h) usage 0 ;;
            v)
                verbose=true
                send_opts="-v -w"
                recv_opts="-v"
                ;;
            q) quiet=true ;;
            k) keep=$OPTARG ;;
            p) port=$OPTARG ;;
            t) tag=$OPTARG ;;
            d) dateopts=$OPTARG ;;
            s) tossh=true ;;
            r) fromssh=true ;;
            C) sshcompress="-C " ;;
            R) reinit="-R " ;;
            I) intermediate="-I " ;;
            g) gpgid="$OPTARG" ;;
            *) usage 1 ;;
        esac
    done
    shift $((OPTIND-1))
    date="$(date $dateopts)"
    $tossh && $fromssh && die 1 "-s and -r are mutually exclusive"
    if ! $tossh && [[ -n $gpgid ]] ; then
        die 1 "-g can only be used with -s"
    fi

    ###
    ### parse src & dest host/fs info
    ###
    # fail if there's ever >1 colon
    if [[ $1 =~ :.*: || $2 =~ :.*: ]] ; then
        die 1 "invalid fsspec: '$1' or '$2'"
    fi

    # fail if src or dest isn't specified
    if [[ -z $1 || -z $2 ]] ; then
        usage 1
    fi
    src="$1"
    dest="$2"

    ###
    ### ssh mode - output snaps from local fs to ssh or read snaps from ssh to local fs
    if $tossh ; then
        log "sending from local zfs filesystem to SSH server"

        # make sure src exists
        if [[ $src =~ : ]] ; then
            die 1 "$src must be a local zfs filesystem"
        elif [[ $(ZFS "" list -H -o name "$src" 2>/dev/null) != $src ]] ; then
            die 1 "$src must be a local zfs filesystem"
        fi

        # split dest to components
        if [[ $dest =~ : ]] ; then
            desthost="${dest%:*}"
            destpath="${dest#*:}"
        else
            die 1 "$dest must be ssh host:path"
        fi

        # get the last src component
        srcbase="${src##*/}"

        ###
        ### create new snapshot on src
        ###
        snap="${tag}_$date"
        cur="$src@$snap"
        ZFS "$srchost" snapshot -r "$cur" || die $? "zfs snapshot failed"

        ###
        ### get newest snapshot on dest - it must exist on src
        ###
        #last="$(ZFS "$desthost" list -d 1 -t snapshot -H -S creation -o name $destfs/$srcbase | head -n1 | cut -f2 -d@)"
        last="$(ssh "$desthost" zfslast)"

        ###
        ### send
        ###
        # refuse to send without a valid .last maker
        if [[ -z $last ]] ; then
            die 1 "ssh path contains no .last file"
        # special case: tagged snapshots exist on dest, but src has rotated through all
        elif ! ZFS "$srchost" list $src@$last &>/dev/null ; then
            die 1 "no incremental path from from $src to $dest"
        # normal case: send incremental
        else
            log "sending $([[ -n $gpgid ]] && echo "encrypted ")incremental snapshot from $src to $dest (${last#${tag}_}..${cur#*@${tag}_})"
            #ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs incremental send failed"
            if [[ -n $gpgid ]] ; then
                ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" \
                    | gpg --trust-model always --encrypt --recipient "$gpgid" \
                    | ssh "$desthost" zfswrite "${tag}_$date.zfssnap.gpg" \
                    || die $? "zfs incremental send failed"
                ssh "$desthost" zfslast "$snap"
            else
                ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" \
                    | ssh "$desthost" zfswrite "${tag}_$date.zfssnap" \
                    || die $? "zfs incremental send failed"
                ssh "$desthost" zfslast "$snap"
            fi
        fi

        exit
    elif $fromssh ; then
        log "receving from SSH server to local zfs filesystem"

        # make sure dest exists
        if [[ $dest =~ : ]] ; then
            die 1 "$dest must be a local zfs filesystem"
        elif [[ $(ZFS "" list -H -o name "$dest" 2>/dev/null) != $dest ]] ; then
            die 1 "$dest must be a local zfs filesystem"
        fi

        # split src into components
        if [[ $src =~ : ]] ; then
            srchost="${src%:*}"
            srcpath="${src#*:}"
        else
            die 1 "$src must be ssh host:path"
        fi

        ###
        ### receive
        ###
        log "receiving incremental snapshot from $src to $dest"
        #ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs incremental send failed"
        for file in $(ssh "$srchost" zfsfind | sort) ; do
            log "receiving $file from $srchost"
            if [[ $file =~ \.gpg$ ]] ; then
                ssh "$srchost" zfsget "$file" | gpg | ZFS "$desthost" receive $recv_opts -Fue "$dest" \
                    && ssh "$srchost" rm "$file"
            else
                ssh "$srchost" zfsget "$file" | ZFS "$desthost" receive $recv_opts -Fue "$dest" \
                    && ssh "$srchost" rm "$file"
            fi
        done

        exit
    fi

    # discard anything before a colon to get the fs
    srcfs="${src#*:}"
    destfs="${dest#*:}"

    # iff there is a colon, discard everything after it to get the host
    [[ $src =~ : ]] && srchost="${src%:*}"
    [[ $dest =~ : ]] && desthost="${dest%:*}"

    # get the last src component
    srcbase="${srcfs##*/}"

    # ensure the destination fs exists before proceeding
    if [[ $(ZFS "$desthost" list -H -o name "$destfs" 2>/dev/null) != $destfs ]] ; then
        die 1 "destination fs '$destfs' doesn't exist"
    fi

    ###
    ### create new snapshot on src
    ###
    cur="$srcfs@${tag}_$date"
    ZFS "$srchost" snapshot -r "$cur" || die $? "zfs snapshot failed"

    ###
    ### get newest snapshot on dest - it must exist on src
    ###
    last="$(ZFS "$desthost" list -d 1 -t snapshot -H -S creation -o name $destfs/$srcbase | head -n1 | cut -f2 -d@)"

    ###
    ### send & receive
    ###
    # 1st time: send full snapshot
    if [[ -z $last ]] ; then
        log "sending full recursive snapshot from $src to $dest"
        ZFS "$srchost" send $send_opts $reinit "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs full send failed"
    # special case: tagged snapshots exist on dest, but src has rotated through all
    elif ! ZFS "$srchost" list $srcfs@$last &>/dev/null ; then
        die 1 "no incremental path from from $src to $dest"
    # normal case: send incremental
    else
        log "sending incremental snapshot from $src to $dest (${last#${tag}_}..${cur#*@${tag}_})"
        ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs incremental send failed"
    fi

    ###
    ### clean up old snapshots
    ###
    for snap in $(ZFS "$srchost" list -d 1 -t snapshot -H -S creation -o name $srcfs \
              | grep -F "@${tag}_" | cut -f2 -d@ | tail -n+$((keep+1)) ) ;
    do
        ZFS "$srchost" destroy -r $srcfs@$snap
    done
    
) 9>/var/lock/bashclub-zfs.lock