diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0c63957 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +ZPOOL_SIZE_SUM_BYTES diff --git a/README.md b/README.md index 9102fd2..9b19134 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Following settings are made: - Disable `pve-enterprise` repo - Add `pve-no-subscription` repo - Upgrade system to latest version -- Install basic tools: `vim ifupdown2 net-tools dnsutils ethtool git curl unzip screen iftop lshw smartmontools nvme-cli lsscsi sysstat zfs-auto-snapshot htop mc rpl` -- Configure snapshot retention for `zfs-auto-snapshot` -- Set limits for level 1 arc (`zfs_arc_min` and `zfs_arc_max`) +- Install basic tools: `sudo vim ifupdown2 net-tools dnsutils ethtool git curl unzip screen iftop lshw smartmontools nvme-cli lsscsi sysstat zfs-auto-snapshot htop mc rpl` +- Configure snapshot retention for `zfs-auto-snapshot` interactively +- Calculates limits for level 1 arc (`zfs_arc_min` and `zfs_arc_max`) based on all configured zpools +- Configure backup of `/etc` folder to new zfs dataset on `rpool/pveconf` diff --git a/install.sh b/install.sh deleted file mode 100644 index 83cc2a6..0000000 --- a/install.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Define basic tools to install -TOOLS="vim ifupdown2 net-tools dnsutils ethtool git curl unzip screen iftop lshw smartmontools nvme-cli lsscsi sysstat zfs-auto-snapshot htop mc rpl" - -# Define zfs-auto-snapshot retention policy -SNAP_FREQUENT=8 -SNAP_HOURLY=48 -SNAP_DAILY=31 -SNAP_WEEKLY=8 -SNAP_MONTHLY=3 - -# Define zfs arcsize in Megabytes (MB) -ZFS_ARC_MIN=128 -ZFS_ARC_MAX=256 - -# disable pve-enterprise repo and add pve-no-subscription repo -mv /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/pve-enterprise.list.bak -echo "deb http://download.proxmox.com/debian/pve buster pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list -apt update - -# update system and install basic tools -DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt -y -qq dist-upgrade -DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt -y -qq install $TOOLS - -# configure zfs-auto-snapshot -rpl "keep=4" "keep=$SNAP_FREQUENT" /etc/cron.d/zfs-auto-snapshot -rpl "keep=24" "keep=$SNAP_HOURLY" /etc/cron.hourly/zfs-auto-snapshot -rpl "keep=31" "keep=$SNAP_DAILY" /etc/cron.hourly/zfs-auto-snapshot -rpl "keep=8" "keep=$SNAP_WEEKLY" /etc/cron.weekly/zfs-auto-snapshot -rpl "keep=8" "keep=$SNAP_MONTHLY" /etc/cron.monthly/zfs-auto-snapshot - -# set zfs_arc_limits -ZFS_ARC_MIN_BYTES=$(($ZFS_ARC_MIN*1024*1024)) -ZFS_ARC_MAX_BYTES=$(($ZFS_ARC_MAX*1024*1024)) - -echo $ZFS_ARC_MIN_BYTES > /sys/module/zfs/parameters/zfs_arc_min -echo $ZFS_ARC_MAX_BYTES > /sys/module/zfs/parameters/zfs_arc_max - -cat << EOF > /etc/modprobe.d/zfs.conf -options zfs zfs_arc_min=$ZFS_ARC_MIN_BYTES -options zfs zfs_arc_min=$ZFS_ARC_MAX_BYTES -EOF -update-initramfs -u -k all diff --git a/proxmox-zfs-postinstall.sh b/proxmox-zfs-postinstall.sh new file mode 100644 index 0000000..6465b2c --- /dev/null +++ b/proxmox-zfs-postinstall.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +###### CONFIG SECTION ###### + +# Define basic tools to install +TOOLS="sudo vim ifupdown2 net-tools dnsutils ethtool git curl unzip screen iftop lshw smartmontools nvme-cli lsscsi sysstat zfs-auto-snapshot htop mc rpl lsb-release" + +#### PVE CONF BACKUP CONFIGURATION #### + +# Define target dataset for backup of /etc +# IMPORTANT NOTE: Don't type in the leading /, this will be set where needed +PVE_CONF_BACKUP_TARGET=rpool/pveconf + +# Define timer for your backup cronjob (default: every 15 minutes fron 3 through 59) +PVE_CONF_BACKUP_CRON_TIMER="3/15 * * * *" + + +###### SYSTEM INFO AND INTERACTIVE CONFIGURATION SECTION ###### + +ROUND_FACTOR=512 + +roundup(){ + echo $(((($1 + $ROUND_FACTOR) / $ROUND_FACTOR) * $ROUND_FACTOR)) +} + +roundoff(){ + echo $((($1 / $ROUND_FACTOR) * $ROUND_FACTOR)) +} + +#### L1ARC SIZE CONFIGURATION #### + +# get total size of all zpools +ZPOOL_SIZE_SUM_BYTES=0 +for line in $(zpool list -o size -Hp); do ZPOOL_SIZE_SUM_BYTES=$(($ZPOOL_SIZE_SUM_BYTES+$line)); done + +# get information about available ram +MEM_TOTAL_BYTES=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) * 1024)) + +# get values if defaults are set +ARC_MAX_DEFAULT_BYTES=$(($MEM_TOTAL_BYTES / 2)) +ARC_MIN_DEFAULT_BYTES=$(($MEM_TOTAL_BYTES / 32)) + +# get current settings +ARC_MIN_CUR_BYTES=$(cat /sys/module/zfs/parameters/zfs_arc_min) +ARC_MAX_CUR_BYTES=$(cat /sys/module/zfs/parameters/zfs_arc_max) + +# calculate suggested l1arc sice +ZFS_ARC_MIN_MEGABYTES=$(roundup $(($ZPOOL_SIZE_SUM_BYTES / 2048 / 1024 / 1024))) +ZFS_ARC_MAX_MEGABYTES=$(roundoff $(($ZPOOL_SIZE_SUM_BYTES / 1024 / 1024 / 1024))) + +echo -e "######## CONFIGURE ZFS L1ARC SIZE ########\n" +echo "System Summary:" +echo -e "\tSystem Memory:\t\t$(($MEM_TOTAL_BYTES / 1024 / 1024))\tMB" +echo -e "\tZpool size (sum):\t$(($ZPOOL_SIZE_SUM_BYTES / 1024 / 1024))\tMB" +echo -e "Calculated l1arc if set to defaults:" +if [ $ARC_MIN_DEFAULT_BYTES -lt 33554432 ]; then + echo -e "\tDefault zfs_arc_min:\t32\tMB" +else + echo -e "\tDefault zfs_arc_min:\t$(($ARC_MIN_DEFAULT_BYTES / 1024 / 1024))\tMB" +fi +echo -e "\tDefault zfs_arc_max:\t$(($ARC_MAX_DEFAULT_BYTES / 1024 / 1024))\tMB" +echo -e "Current l1arc configuration:" +if [ $ARC_MIN_CUR_BYTES -gt 0 ]; then + echo -e "\tCurrent zfs_arc_min:\t$(($ARC_MIN_CUR_BYTES / 1024 / 1024))\tMB" +else + echo -e "\tCurrent zfs_arc_min:\t0" +fi +if [ $ARC_MAX_CUR_BYTES -gt 0 ]; then + echo -e "\tCurrent zfs_arc_max:\t$(($ARC_MAX_CUR_BYTES / 1024 / 1024))\tMB" +else + echo -e "\tCurrent zfs_arc_max:\t0" +fi +echo -e "Note: If your current values are 0, the calculated values above will apply." +echo "" +echo -e "The l1arc cache will be set relative to the size (sum) of your zpools by policy" +echo -e "zfs_arc_min:\t\t\t$(($ZFS_ARC_MIN_MEGABYTES))\tMB\t\t= 512 MB RAM per 1 TB ZFS storage (round off in 512 MB steps)" +echo -e "zfs_arc_max:\t\t\t$(($ZFS_ARC_MAX_MEGABYTES))\tMB\t\t= 1 GB RAM per 1 TB ZFS storage (round up in 512 MB steps)" +echo "" +RESULT=not_set +while [ "$(echo $RESULT | awk '{print tolower($0)}')" != "y" ] && [ "$(echo $RESULT | awk '{print tolower($0)}')" != "n" ] && [ "$(echo $RESULT | awk '{print tolower($0)}')" != "" ]; do + echo "If you want to apply the values by script policy type 'y', type 'n' to adjust the values yourself [Y/n]?" + read + RESULT=${REPLY} +done +if [[ "$(echo $RESULT | awk '{print tolower($0)}')" == "n" ]]; then + echo "Please type in the desired value in MB for 'zfs_arc_min' [$(($ZFS_ARC_MIN_MEGABYTES))]:" + read + if [[ ${REPLY} -gt 0 ]]; then + ZFS_ARC_MIN_MEGABYTES=$((${REPLY})) + fi + echo "Please type in the desired value in MB for 'zfs_arc_max' [$(($ZFS_ARC_MAX_MEGABYTES))]:" + read + if [[ ${REPLY} -gt 0 ]]; then + ZFS_ARC_MAX_MEGABYTES=$((${REPLY})) + fi +fi + +#### SWAPPINESS #### + +echo -e "######## CONFIGURE SWAPPINESS ########\n" +SWAPPINESS=$(cat /proc/sys/vm/swappiness) +echo "The current swappiness is configured to '$SWAPPINESS %' of free memory until using swap." +read -p "If you want to change the swappiness, please type in the percentage as number (0 = diasbled):" user_input +if echo "$user_input" | grep -qE '^[0-9]+$'; then + echo "Changing swappiness from '$SWAPPINESS %' to '$user_input %'" + SWAPPINESS=$user_input +else + echo "No input - swappiness unchanged at '$SWAPPINESS %'." +fi + +#### ZFS AUTO SNAPSHOT CONFIGURATION #### + +# get information about zfs-auto-snapshot and ask for snapshot retention +declare -A auto_snap_keep=( ["frequent"]="8" ["hourly"]="48" ["daily"]="31" ["weekly"]="8" ["monthly"]="3" ) +dpkg -l zfs-auto-snapshot > /dev/null + +if [ $? -eq 0 ]; then + echo "'zfs-auto-snapshot' already installed. Reading config..." + for interval in "${!auto_snap_keep[@]}"; do + if [[ "$interval" == "frequent" ]]; then + auto_snap_keep[$interval]=$(cat /etc/cron.d/zfs-auto-snapshot | grep keep | cut -d' ' -f19 | cut -d '=' -f2) + else + auto_snap_keep[$interval]=$(cat /etc/cron.$interval/zfs-auto-snapshot | grep keep | cut -d' ' -f6 | cut -d'=' -f2) + fi + done +else + echo "'zfs-auto-snapshot' not installed yet, using script defaults..." +fi +echo -e "######## CONFIGURE ZFS AUTO SNAPSHOT ########\n" +for interval in "${!auto_snap_keep[@]}"; do + read -p "Please set how many $interval snapshots to keep (current: keep=${auto_snap_keep[$interval]})" user_input + if echo "$user_input" | grep -qE '^[0-9]+$'; then + echo "Changing $interval from ${auto_snap_keep[$interval]} to $user_input" + auto_snap_keep[$interval]=$user_input + else + echo "No input - $interval unchanged at ${auto_snap_keep[$interval]}." + fi +done + +###### INSTALLER SECTION ###### + +# disable pve-enterprise repo and add pve-no-subscription repo +if [[ "$(uname -r)" == *"-pve" ]]; then + echo "Deactivating pve-enterprise repository" + mv /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/pve-enterprise.list.bak > /dev/null 2>&1 + echo "Activating pve-no-subscription repository" + echo "deb http://download.proxmox.com/debian/pve buster pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list +fi +echo "Getting latest package lists" +apt update > /dev/null 2>&1 + +# update system and install basic tools +echo "Upgrading system to latest version - Depending on your version this could take a while..." +DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt -y -qq dist-upgrade > /dev/null 2>&1 +echo "Installing toolset - Depending on your version this could take a while..." +DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt -y -qq install $TOOLS > /dev/null 2>&1 + +# configure zfs-auto-snapshot +for interval in "${!auto_snap_keep[@]}"; do + echo "Setting zfs-auto-snapshot retention: $interval = ${auto_snap_keep[$interval]}" + if [[ "$interval" == "frequent" ]]; then + CURRENT=$(cat /etc/cron.d/zfs-auto-snapshot | grep keep | cut -d' ' -f19 | cut -d '=' -f2) + if [[ "${auto_snap_keep[$interval]}" != "$CURRENT" ]]; then + rpl "keep=$CURRENT" "keep=${auto_snap_keep[$interval]}" /etc/cron.d/zfs-auto-snapshot > /dev/null 2>&1 + fi + else + CURRENT=$(cat /etc/cron.$interval/zfs-auto-snapshot | grep keep | cut -d' ' -f6 | cut -d'=' -f2) + if [[ "${auto_snap_keep[$interval]}" != "$CURRENT" ]]; then + rpl "keep=$CURRENT" "keep=${auto_snap_keep[$interval]}" /etc/cron.$interval/zfs-auto-snapshot > /dev/null 2>&1 + fi + fi +done + +echo "Configuring swappiness" +echo "vm.swappiness=$SWAPPINESS" > /etc/sysctl.d/swappiness.conf +sysctl -w vm.swappiness=$SWAPPINESS + +echo "Configuring pve-conf-backup" +# create backup jobs of /etc +zfs list $PVE_CONF_BACKUP_TARGET > /dev/null 2>&1 +if [ $? -ne 0 ]; then + zfs create $PVE_CONF_BACKUP_TARGET +fi +echo "$PVE_CONF_BACKUP_CRON_TIMER root rsync -va --delete /etc /$PVE_CONF_BACKUP_TARGET > /$PVE_CONF_BACKUP_TARGET/pve-conf-backup.log" > /etc/cron.d/pve-conf-backup + +ZFS_ARC_MIN_BYTES=$((ZFS_ARC_MIN_MEGABYTES * 1024 *1024)) +ZFS_ARC_MAX_BYTES=$((ZFS_ARC_MAX_MEGABYTES * 1024 *1024)) + +echo "Adjusting ZFS level 1 arc" +echo $ZFS_ARC_MIN_BYTES > /sys/module/zfs/parameters/zfs_arc_min +echo $ZFS_ARC_MAX_BYTES > /sys/module/zfs/parameters/zfs_arc_max + +cat << EOF > /etc/modprobe.d/zfs.conf +options zfs zfs_arc_min=$ZFS_ARC_MIN_BYTES +options zfs zfs_arc_min=$ZFS_ARC_MAX_BYTES +EOF + +echo "Updating initramfs - This will take some time..." +update-initramfs -u -k all > /dev/null 2>&1 + +echo "Proxmox postinstallation finished!" \ No newline at end of file