zsync/bashclub-zsync/usr/bin/bashclub-zsync

#!/bin/bash
#
# bashclub zfs replication script
# Author: (C) 2023 Thorsten Spille <thorsten@spille-edv.de>

PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"

prog="$(basename $0)"
zfs=$(which zfs)
ssh=$(which ssh)
debug=

#### default config file, can be changed with parameter -c
conf=/etc/bashclub/zsync.conf

#### default values in config file
# replication target on local machine 
target=pool/dataset

# ssh address of remote machine
source=user@host

# ssh port of remote machine
sshport=22

# zfs user parameter to identify filesystems/volumes to replicate
tag=bashclub:zsync

# if $tag=subvols, which source to filter: "inherited" or "inherited|received"
subvol_source="inherited|received"

# pipe separated list of snapshot name filters
snapshot_filter="hourly|daily|weekly|monthly"

# minimum count of snapshots per filter to keep
min_keep=3

usage() {
	cat >&2 <<-EOF
	usage: $prog [-h] [-d] [-c CONFIG]
	  creates a mirrored replication of configured zfs filesystems / volumes
    -c CONFIG    Configuration file for this script
    -d           Debug mode
  ---------------------------------------------------------------------------
    (C) 2023     by Spille IT Solutions for bashclub (github.com/bashclub)
                 Author: Thorsten Spille <thorsten@spille-edv.de>
  ---------------------------------------------------------------------------
	EOF
	exit $1
}

while getopts "hdc:" opt; do
  case $opt in
    h) usage 0 ;;
    c) conf=$OPTARG ;;
    d) debug=-v ;;
    *) usage 1 ;;
  esac
done
shift $((OPTIND-1))

function log() {
    echo -e "$(date +'%b %d %T') $1"
}

# load config file
if [ -f $conf ]; then
    log "Reading configuration $conf"
    source $conf
else
    mkdir -p $(dirname $conf)
    cat << EOF > $conf
target=$target
source=$source
sshport=$sshport
tag=$tag
subvol_source="$subvol_source"
snapshot_filter="$snapshot_filter"
min_keep=$min_keep
EOF
    log "Initial config file created. Please adjust and restart script. Exiting..."
    usage 0
fi

if [[ $source == "" ]]; then
    log "source is empty, switching to local mode."
    ssh=
    sshport=
    log "Configuration:\n\ttarget=$target\n\ttag=$tag\n\tsubvol_source=$subvol_source\n\tsnapshot_filter=$snapshot_filter\n"
else
    sshport=-p$sshport
    log "Configuration:\n\ttarget=$target\n\tsource=$source\n\tsshport=$sshport\n\ttag=$tag\n\tsubvol_source=$subvol_source\n\tsnapshot_filter=$snapshot_filter\n"
fi


local_aes=$(/usr/bin/grep -m1 -o aes /proc/cpuinfo | uniq)
remote_aes=$($ssh $source $sshport "/usr/bin/grep -m1 -o aes /proc/cpuinfo | uniq")


if [[ $local_aes == "aes" ]] && [[ $remote_aes == "aes" ]]; then
    sshcipher=-caes256-gcm@openssh.com
fi


# query source datasets/subvols to replicate
IFS=$'\n'
for zvol in $($ssh $sshcipher $sshport $source "zfs get -H -o name,value,source -t filesystem,volume $tag"); do
    name=$(echo $zvol | cut -f1)
    if [[ "$(echo $zvol | cut -f2)" == "subvols" ]] && [[ $(echo $zvol | grep -E $subvol_source) ]]; then
        log "Including $name"
        syncvols=("${syncvols[@]}" "$name")
    elif [[ "$(echo $zvol | cut -f2)" == "all" ]];then
        log "Including $name"
        syncvols=("${syncvols[@]}" "$name")
    else
        log "Excluding $name"
    fi
done


for name in "${syncvols[@]}"; do
    IFS=$' '
    if [ $($zfs list -H -s creation -t snapshot $target/$name > /dev/null 2>&1 ; echo $?) -gt 0 ]; then
        # create parent datasets
        prefix=""
        for part in $(echo $target/$(echo $name | cut -d'/' -f1) | sed "s/\// /g"); do
            if [ $($zfs list $prefix$part > /dev/null 2>&1 ; echo $?) -gt 0 ]; then
                log "Creating $prefix$part"
                if [[ $prefix/$part == $target ]]; then
                    autosnap=-ocom.sun:auto-snapshot=false
                fi
                $zfs create $autosnap -p $prefix$part
            else
                if [[ $prefix/$part == $target ]] && [[ $(zfs get -H -o value,source com.sun:auto-snapshot $prefix/$part) != "false local" ]]; then
                    $zfs set com.sun:auto-snapshot=false $prefix/$part
                fi
            fi
            prefix="$prefix$part/"
        done

        # start initial replication
        IFS=$'\n'
        for snap in $($ssh $sshcipher $sshport $source "zfs list -H -t snapshot -o name -S creation $name | grep -E \"$snapshot_filter\" | tail -1"); do
            log "Start initial replication: $snap => $target/$(echo $name | cut -d'/' -f1)"
            $ssh $ssh_ipher $sshport $source "zfs send -w -p $debug $snap" | $zfs receive -x mountpoint -x canmount -x $tag -x com.sun:auto-snapshot $debug -dF $target/$(echo $name | cut -d'/' -f1)
        done
    fi

    # replicate incremental
    guid=$($zfs list -H -o guid -s creation -t snapshot $target/$name | tail -1)
    last=$($ssh $sshcipher $sshport $source "zfs list -H -o name,guid -t snapshot $name | grep $guid | tail -1 | cut -f1")
    IFS=$'\n'
    for snap in $($ssh $sshcipher $sshport $source "zfs list -H -o name,guid -s creation -t snapshot $name | grep -E \"$snapshot_filter\" | grep --after-context=200 $guid | grep -v $guid | cut -f1"); do
        log "Replicating delta of $last => $snap to $target/$name"
        $ssh $sshcipher $sshport $source "zfs send -w $debug -i $last $snap" | zfs receive -x mountpoint -x canmount -x $tag -x com.sun:auto-snapshot -F $debug $target/$name
        last=$snap
    done

    # cleanup old snapshots
    filter=$(echo -e $snapshot_filter | sed "s/|/\n/g")
    IFS=$'\n'
    for interval in $filter ; do
        guid=$($ssh $sshcipher $sshport $source "zfs list -H -o guid,name -S creation -t snapshot $name | grep $interval | cut -f1 | tail -1")
        if [[ "$(echo -e "$guid" | sed 's/\n//g')" != "" ]]; then
            snaps_to_delete=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | grep $interval | grep --after-context=200 $guid | grep -v $guid | cut -f1)
            delete_count=$(echo $snaps_to_delete | wc -l)
            for snap in $snaps_to_delete; do
                if [[ $delete_count -gt $min_keep ]]; then
                    log "Deleting $snap"
                    if [[ $debug == "-v" ]]; then log "[DEBUG] delete_count=$delete_count, min_keep=$min_keep"; fi
                    $zfs destroy $debug $snap
                    delete_count=$(expr $delete_count - 1)
                else
                    if [[ $debug == "-v" ]]; then log "[DEBUG] Skipping deletion of $snap. delete_count=$delete_count, min_keep=$min_keep"; fi
                fi
            done
        fi
    done
done