diff --git a/README.md b/README.md index f946f94..f35c721 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ZFS replication script by Thorsten Spille #### Download and make executable ~~~ -wget -q --no-cache -O /usr/bin/bashclub-zsync https://git.bashclub.org/bashclub/zsync/raw/branch/main/bashclub-zsync/usr/bin/bashclub-zsync +wget -q --no-cache -O /usr/bin/bashclub-zsync https://git.bashclub.org/bashclub/zsync/raw/branch/dev/bashclub-zsync/usr/bin/bashclub-zsync chmod +x /usr/bin/bashclub-zsync bashclub-zsync ~~~ diff --git a/bashclub-zsync/usr/bin/bashclub-zsync b/bashclub-zsync/usr/bin/bashclub-zsync index 656dc5b..f7f3ab7 100644 --- a/bashclub-zsync/usr/bin/bashclub-zsync +++ b/bashclub-zsync/usr/bin/bashclub-zsync @@ -16,6 +16,7 @@ wc=$(which wc) tr=$(which tr) sed=$(which sed) zfs_auto_snapshot=$(which zfs-auto-snapshot) +rc=0 debug= #### default config file, can be changed with parameter -c @@ -40,7 +41,7 @@ snapshot_filter="hourly|daily|weekly|monthly" # minimum count of snapshots per filter to keep min_keep=3 -# number of zfs snapshots to keep on source (0 = snapshot function disabled) +# number of zfs snapshots to keep on source (0 or 1 = snapshot function disabled) zfs_auto_snapshot_keep=0 # make snapshot via zfs-auto-snapshot before replication @@ -76,7 +77,7 @@ function log() { # load config file if [ -f $conf ]; then - log "Reading configuration $conf" + log "[INFO] Reading configuration $conf" source $conf else mkdir -p $(dirname $conf) @@ -90,18 +91,18 @@ min_keep=$min_keep zfs_auto_snapshot_keep=$zfs_auto_snapshot_keep zfs_auto_snapshot_label=$zfs_auto_snapshot_label EOF - log "Initial config file created. Please adjust and restart script. Exiting..." + log "[INFO] Initial config file created. Please adjust and restart script. Exiting..." usage 0 fi if [[ $source == "" ]]; then - log "source is empty, switching to local mode." + log "[INFO] source is empty, switching to local mode." ssh= sshport= - log "Configuration:\n\ttarget=$target\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n" + log "[INFO] Configuration:\n\ttarget=$target\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\nzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\nzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n" else sshport=-p$sshport - log "Configuration:\n\ttarget=$target\n\tsource=$source\n\tsshport=$sshport\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n" + log "[INFO] Configuration:\n\ttarget=$target\n\tsource=$source\n\tsshport=$sshport\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\nzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\nzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n" fi local_os_id=$($grep -E "^ID=" /etc/os-release | $cut -d'=' -f2) @@ -127,8 +128,8 @@ else sshcipher=-cchacha20-poly1305@openssh.com fi -include_list="Included datasets:\n" -exclude_list="Excluded datasets:\n" +include_list="[INFO] Included datasets:\n" +exclude_list="[INFO] Excluded datasets:\n" IFS=$'\n' for zvol in $($ssh $sshcipher $sshport $source "zfs get -H -o name,value,source -t filesystem,volume $tag"); do name=$(echo $zvol | $cut -f1) @@ -155,7 +156,7 @@ else fi fi -if [ $zfs_auto_snapshot_keep -gt 0 ]; then +if [ $zfs_auto_snapshot_keep -gt 1 ]; then log "[DEBUG] Running zfs-auto-snapshot" $ssh $sshcipher $sshport $source "which zfs-auto-snapshot > /dev/null || exit 0 ; zfs-auto-snapshot --quiet --syslog --label=$zfs_auto_snapshot_label --keep=$zfs_auto_snapshot_keep //" if [[ $snapshot_filter == "" ]]; then @@ -166,8 +167,9 @@ if [ $zfs_auto_snapshot_keep -gt 0 ]; then fi for name in "${syncvols[@]}"; do - log "Replicate $name" - if [[ $($ssh $sshcipher $sshport $source zfs get -H -o value type $name) == "filesystem" ]]; then + log "[INFO] Replicate $name" + fstype=$($ssh $sshcipher $sshport $source zfs get -H -o value type $name) + if [[ $fstype == "filesystem" ]]; then mp=-xmountpoint cm=-ocanmount=noauto else @@ -175,7 +177,7 @@ for name in "${syncvols[@]}"; do cm= fi - if [[ $($ssh $sshcipher $sshport $source "zfs list -H -t snapshot -o name -S creation $name 2>/dev/null | grep -E \"$snapshot_filter\" | wc -l | tr -d ' '") -gt 0 ]]; then + if [[ $($ssh $sshcipher $sshport $source "zfs list -H -t snapshot -o name -S creation $name 2>/dev/null | grep -E \"@.*($snapshot_filter)\" | wc -l | tr -d ' '") -gt 0 ]]; then IFS=$' ' if ! $zfs list -H $target/$name > /dev/null 2>&1 ; then if [[ $debug == "-v" ]]; then log "[DEBUG] $target/$name does not exist"; fi @@ -183,7 +185,7 @@ for name in "${syncvols[@]}"; do for part in $(echo $target/$(echo $name | $cut -d'/' -f1) | $sed "s/\// /g"); do if [ $($zfs list $prefix$part > /dev/null 2>&1 ; echo $?) -gt 0 ]; then if [[ $debug == "-v" ]]; then log "[DEBUG] $prefix$part does not exist"; fi - log "Creating $prefix$part" + log "[INFO] Creating $prefix$part" $zfs create -o canmount=noauto $autosnap -p $prefix$part fi prefix="$prefix$part/" @@ -191,38 +193,50 @@ for name in "${syncvols[@]}"; do if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start initial replication"; fi IFS=$'\n' - for snap in $($ssh $sshcipher $sshport $source "zfs list -H -t snapshot -o name -S creation $name | grep -E \"$snapshot_filter\" | tail -1"); do - log "Start initial replication: $snap => $target/$(echo $name | $cut -d'/' -f1)" + for snap in $($ssh $sshcipher $sshport $source "zfs list -H -t snapshot -o name -S creation $name | grep -E \"@.*($snapshot_filter)\" | tail -1"); do + log "[INFO] Start initial replication: $snap => $target/$(echo $name | $cut -d'/' -f1)" $ssh $sshcipher $sshport $source "zfs send -w -p $debug $snap" | $zfs receive $mp $cm -x $tag -x com.sun:auto-snapshot $debug -dF $target/$(echo $name | $cut -d'/' -f1) + if [ $? -gt $rc ]; then rc=1; log "[ERROR] initial replication to $target/$name failed."; fi done fi if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start incremental replication"; fi guid=$($zfs list -H -o guid -s creation -t snapshot $target/$name | $tail -1) - last=$($ssh $sshcipher $sshport $source "zfs list -H -o name,guid -t snapshot $name | grep $guid | tail -1 | cut -f1") - IFS=$'\n' - if [[ $($zfs get -H -o value canmount $target/$name) != "noauto" ]]; then - $zfs set canmount=noauto $target/$name + if [[ $guid != "" ]]; then + last=$($ssh $sshcipher $sshport $source "zfs list -H -o name,guid -t snapshot $name | grep $guid | tail -1 | cut -f1") + if [[ $last != "" ]]; then + IFS=$'\n' + if [[ $fstype == "filesystem" ]] && [[ $($zfs get -H -o value canmount $target/$name) != "noauto" ]]; then + $zfs set canmount=noauto $target/$name + fi + for snap in $($ssh $sshcipher $sshport $source "zfs list -H -o name,guid -s creation -t snapshot $name | grep -E \"@.*($snapshot_filter)\" | grep --after-context=200 $guid | grep -v $guid | cut -f1"); do + log "[INFO] Replicating delta of $last => $snap to $target/$name" + $ssh $sshcipher $sshport $source "zfs send -w $debug -i $last $snap" | zfs receive -x $tag -x com.sun:auto-snapshot -F $debug $target/$name + if [ $? -gt $rc ]; then rc=1; log "[ERROR] incremental replication to $target/$name failed."; fi + last=$snap + done + else + log "[ERROR] No matching snapshot (guid: $guid) found on source filesystem. This can be outdated snapshots on target or a previously deleted and new created dataset $name on the source filesystem." + if [ $rc -eq 0 ]; then rc=1; fi + fi + else + log "[ERROR] No snapshot found on $target/$name to add incremental snapshots to. The target dataset (with all children) needs to be deleted and recreated via replication." + if [ $rc -eq 0 ]; then rc=1; fi fi - for snap in $($ssh $sshcipher $sshport $source "zfs list -H -o name,guid -s creation -t snapshot $name | grep -E \"$snapshot_filter\" | grep --after-context=200 $guid | grep -v $guid | cut -f1"); do - log "Replicating delta of $last => $snap to $target/$name" - $ssh $sshcipher $sshport $source "zfs send -w $debug -i $last $snap" | zfs receive -x $tag -x com.sun:auto-snapshot -F $debug $target/$name - last=$snap - done if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start deletion of old snapshots"; fi filter=$(echo -e $snapshot_filter | sed "s/|/\n/g") IFS=$'\n' for interval in $filter ; do if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Checking interval $interval"; fi - guid=$($ssh $sshcipher $sshport $source "zfs list -H -o guid,name -S creation -t snapshot $name | grep -E \"(.*)@(.*)$interval\" | cut -f1 | tail -1") + guid=$($ssh $sshcipher $sshport $source "zfs list -H -o guid,name -S creation -t snapshot $name | grep -E \"@.*$interval\" | cut -f1 | tail -1") if [[ "$(echo -e "$guid" | sed 's/\n//g')" != "" ]]; then - snaps_to_delete=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "(.*)@(.*)$interval" | $grep --after-context=200 $guid | $grep -v $guid | $cut -f1) - snap_count=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "(.*)@(.*)$interval" | $wc -l | $tr -d ' ') + snaps_to_delete=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $grep --after-context=200 $guid | $grep -v $guid | $cut -f1) + snap_count=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $wc -l | $tr -d ' ') for snap in $snaps_to_delete; do if [[ $snap_count -gt $min_keep ]]; then - log "Deleting $snap" + log "[INFO] Deleting $snap" if [[ $debug == "-v" ]]; then log "[DEBUG] $name - snap_count=$snap_count, min_keep=$min_keep"; fi $zfs destroy $debug $snap snap_count=$(expr $snap_count - 1) @@ -236,3 +250,4 @@ for name in "${syncvols[@]}"; do if [[ $debug == "-v" ]]; then log "[DEBUG] $name - No snapshots found with filter $snapshot_filter"; fi fi done +exit $rc \ No newline at end of file