Merge pull request hestiacp#1210 from hestiacp/staging/features

ScIT-Raphael · web-flow · commit 6e9830b12089 · 2020-09-21T09:58:12.000+02:00
hestiacp#1195 ZSTD Support (hestiacp#1202)
diff --git a/bin/v-backup-user b/bin/v-backup-user
@@ -245,7 +245,11 @@ if [ ! -z "$WEB_SYSTEM" ] && [ "$WEB" != '*' ]; then
         set +f
 
         # Backup files
-        tar ${fargs[@]} -cpf- * | gzip -$BACKUP_GZIP - > $tmpdir/web/$domain/domain_data.tar.gz
+        if [ "$BACKUP_MODE" = 'zstd' ]; then
+            tar ${fargs[@]} -cpf- * | zstd -$BACKUP_GZIP - > $tmpdir/web/$domain/domain_data.tar.zst
+        else    
+            tar ${fargs[@]} -cpf- * | gzip -$BACKUP_GZIP - > $tmpdir/web/$domain/domain_data.tar.gz
+        fi
     done
 
     # Print total
@@ -358,10 +362,13 @@ if [ ! -z "$MAIL_SYSTEM" ] && [ "$MAIL" != '*' ]; then
                     tee -a $BACKUP/$user.log
             fi
         done
-
         # Compress archive
         if [ ${#accounts[@]} -gt 0 ]; then
-            tar -cpf- ${accounts[@]} |gzip -$BACKUP_GZIP - > $tmpdir/mail/$domain/accounts.tar.gz
+            if [ "$BACKUP_MODE" = 'zstd' ]; then
+                tar -cpf- ${accounts[@]}  | zstd -$BACKUP_GZIP - > $tmpdir/mail/$domain/accounts.tar.zst
+            else    
+                tar -cpf- ${accounts[@]} | gzip -$BACKUP_GZIP - > $tmpdir/mail/$domain/accounts.tar.gz
+            fi        
         fi
     done
 
@@ -406,7 +413,12 @@ if [ ! -z "$DB_SYSTEM" ] && [ "$DB" != '*' ]; then
         grep "DB='$database'" $conf > hestia/db.conf
 
         dump="$tmpdir/db/$database/$database.$TYPE.sql"
-        dumpgz="$tmpdir/db/$database/$database.$TYPE.sql.gz"
+        if [ "$BACKUP_MODE" = 'zstd' ]; then
+                    dumpgz="$tmpdir/db/$database/$database.$TYPE.sql.zst"
+        else
+            dumpgz="$tmpdir/db/$database/$database.$TYPE.sql.gz"        
+        fi
+
         grants="$tmpdir/db/$database/conf/$database.$TYPE.$DBUSER"
         if [ ! -f "$dumpgz" ]; then
 
@@ -432,7 +444,12 @@ if [ ! -z "$DB_SYSTEM" ] && [ "$DB" != '*' ]; then
             esac
 
             # Compress dump
-            gzip -$BACKUP_GZIP $dump
+            if [ "$BACKUP_MODE" = 'zstd' ]; then
+                zstd -$BACKUP_GZIP $dump
+                rm $dump;
+            else    
+                gzip -$BACKUP_GZIP $dump
+            fi                     
         fi
     done
 
@@ -508,7 +525,12 @@ if [ "$USER" != '*' ]; then
             check_backup_conditions
 
             # Backup files and dirs
-            tar --anchored -cpf- ${fargs[@]} $udir |gzip -$BACKUP_GZIP - > $tmpdir/user_dir/$udir.tar.gz
+            if [ "$BACKUP_MODE" = 'zstd' ]; then
+                tar --anchored -cpf- ${fargs[@]} $udir | zstd -$BACKUP_GZIP - > $tmpdir/user_dir/$udir.tar.zst
+            else    
+                tar --anchored -cpf- ${fargs[@]} $udir | gzip -$BACKUP_GZIP - > $tmpdir/user_dir/$udir.tar.gz
+            fi 
+            
         fi
     done
     set +f
@@ -523,6 +545,9 @@ if [ "$USER" != '*' ]; then
             tee -a $BACKUP/$user.log
     fi
 fi
+if [ "$BACKUP_MODE" = 'zstd' ]; then
+    touch $tmpdir/.zstd
+fi
 
 # Get backup size
 size="$(du -shm $tmpdir |cut -f 1)"
diff --git a/bin/v-extract-fs-archive b/bin/v-extract-fs-archive
@@ -64,7 +64,6 @@ if [ ! -z "$(echo $src_file |egrep -i  '.tgz|.tar.gz')" ]; then
         user_exec tar -tf "$src_file" --no-wildcards "$selected_dir" >/dev/null 2>&1
         rc=$?
     fi
-
 fi
 
 # Extracting bziped archive
@@ -80,6 +79,27 @@ if [ ! -z "$(echo $src_file |egrep -i  '.tbz|.tar.bz')" ]; then
     fi
 fi
 
+# Extracting ZSTD archive
+if [ ! -z "$(echo $src_file |egrep -i  '.tar.zst')" ]; then
+    x='yes'
+    if [ -z "$test" ] || [ "$test" = "no" ]; then
+        user_exec mkdir -p "$dst_dir" >/dev/null 2>&1
+        user_exec tar -I zstd -xf "$src_file" -C "$dst_dir" --no-wildcards "$selected_dir" $tar_strip_level >/dev/null 2>&1
+        rc=$?
+    else
+        user_exec tar -I zstd -tf "$src_file" --no-wildcards "$selected_dir" >/dev/null 2>&1
+        rc=$?
+    fi
+fi
+
+# Extracting gziped file
+if [ ! -z "$(echo $src_file |grep -i  '.zst')" ] && [ -z "$x" ]; then
+    user_exec mkdir -p "$dst_dir" >/dev/null 2>&1
+    user_exec mv "$src_file" "$dst_dir" >/dev/null 2>&1
+    user_exec zstd -d "$dst_dir/$(basename $src_file)" >/dev/null 2>&1
+    rc=$?
+fi
+
 # Extracting gziped file
 if [ ! -z "$(echo $src_file |grep -i  '.gz')" ] && [ -z "$x" ]; then
     user_exec mkdir -p "$dst_dir" >/dev/null 2>&1
diff --git a/bin/v-restore-user b/bin/v-restore-user
@@ -120,12 +120,17 @@ fi
 
 # Set default backup source system
 backup_system="hestia"
+backup_mode="gzip"
 
 # Check if it is a Vesta backup
 if tar -tf "$BACKUP/$backup" ./vesta >/dev/null 2>&1; then
     backup_system="vesta"
 fi
 
+if tar -tf "$BACKUP/$backup" ./.zstd >/dev/null 2>&1; then
+    backup_mode="zstd"
+fi
+
 # Restoring user account
 if [ "$create_user" = 'yes' ]; then
     echo "-- USER --" |tee $tmpdir/restore.log
@@ -170,7 +175,11 @@ if [ "$web" != 'no' ] && [ ! -z "$WEB_SYSTEM" ]; then
 
     # Creating web domain restore list
     backup_domains=$(tar -tf $BACKUP/$backup |grep "^./web")
-    backup_domains=$(echo "$backup_domains" |grep domain_data.tar.gz)
+    if [ "$backup_mode" = "zstd" ]; then
+        backup_domains=$(echo "$backup_domains" |grep domain_data.tar.zst)
+    else
+        backup_domains=$(echo "$backup_domains" |grep domain_data.tar.gz)
+    fi
     backup_domains=$(echo "$backup_domains" |cut -f 3 -d /)
     if [ -z "$web" ] || [ "$web" = '*' ]; then
         domains="$backup_domains"
@@ -307,10 +316,19 @@ if [ "$web" != 'no' ] && [ ! -z "$WEB_SYSTEM" ]; then
         fi
         chmod u+w "$HOMEDIR/$user/web/$domain"
         [[ -d $HOMEDIR/$user/web/$domain/stats ]] && chmod u+w "$HOMEDIR/$user/web/$domain/stats"
-        user_exec tar -xzpf $tmpdir/web/$domain/domain_data.tar.gz \
-            -C "$HOMEDIR/$user/web/$domain/" \
-            --anchored \
-            --exclude='logs/*'
+    
+        if [ "$backup_mode" = "zstd" ]; then
+            user_exec tar -I zstd -xpf $tmpdir/web/$domain/domain_data.tar.zst \
+                -C "$HOMEDIR/$user/web/$domain/" \
+                --anchored \
+                --exclude='logs/*'
+        else
+            user_exec tar -xzpf $tmpdir/web/$domain/domain_data.tar.gz \
+                -C "$HOMEDIR/$user/web/$domain/" \
+                --anchored \
+                --exclude='logs/*'
+        fi
+        
         if [ "$?" -ne 0 ]; then
             rm -rf $tmpdir
             error="Can't unpack $domain data tarball"
@@ -573,22 +591,39 @@ if [ "$mail" != 'no' ] && [ ! -z "$MAIL_SYSTEM" ]; then
         format_domain_idn
 
         # Restoring emails
-        if [ -e "$tmpdir/mail/$domain/accounts.tar.gz" ]; then
-            chmod u+w "$HOMEDIR/$user/mail/$domain_idn"
-            $BIN/v-extract-fs-archive "$user" "$tmpdir/mail/$domain/accounts.tar.gz" "$HOMEDIR/$user/mail/$domain_idn/"
-            if [ "$?" -ne 0 ]; then
-                rm -rf $tmpdir
-                error="Can't unpack $domain mail account container"
-                echo "$error" |$SENDMAIL -s "$subj" $email $notify
-                sed -i "/ $user /d" $HESTIA/data/queue/backup.pipe
-                check_result "$E_PARSING" "$error"
+        if [ $BACKUP_MODE = 'zstd' ]; then    
+            if [ -e "$tmpdir/mail/$domain/accounts.tar.zst" ]; then
+                chmod u+w "$HOMEDIR/$user/mail/$domain_idn"
+                $BIN/v-extract-fs-archive "$user" "$tmpdir/mail/$domain/accounts.tar.zst" "$HOMEDIR/$user/mail/$domain_idn/"
+                if [ "$?" -ne 0 ]; then
+                    rm -rf $tmpdir
+                    error="Can't unpack $domain mail account container"
+                    echo "$error" |$SENDMAIL -s "$subj" $email $notify
+                    sed -i "/ $user /d" $HESTIA/data/queue/backup.pipe
+                    check_result "$E_PARSING" "$error"
+                fi
+    
+                # Chowning as owner needs to be user:mail instead of user:user
+                find $HOMEDIR/$user/mail/$domain_idn -user $user \
+                   -exec chown -h $user:mail {} \;
+            fi
+        else
+            if [ -e "$tmpdir/mail/$domain/accounts.tar.gz" ]; then
+                chmod u+w "$HOMEDIR/$user/mail/$domain_idn"
+                $BIN/v-extract-fs-archive "$user" "$tmpdir/mail/$domain/accounts.tar.gz" "$HOMEDIR/$user/mail/$domain_idn/"
+                if [ "$?" -ne 0 ]; then
+                    rm -rf $tmpdir
+                    error="Can't unpack $domain mail account container"
+                    echo "$error" |$SENDMAIL -s "$subj" $email $notify
+                    sed -i "/ $user /d" $HESTIA/data/queue/backup.pipe
+                    check_result "$E_PARSING" "$error"
+                fi
+    
+                # Chowning as owner needs to be user:mail instead of user:user
+                find $HOMEDIR/$user/mail/$domain_idn -user $user \
+                   -exec chown -h $user:mail {} \;
             fi
-
-            # Chowning as owner needs to be user:mail instead of user:user
-            find $HOMEDIR/$user/mail/$domain_idn -user $user \
-               -exec chown -h $user:mail {} \;
         fi
-
         # Chowning mail conf files to exim user
         find $HOMEDIR/$user/conf/mail/$domain_idn -user root \
             -exec chown $exim_user {} \;
@@ -661,7 +696,11 @@ if [ "$db" != 'no' ] && [ ! -z "$DB_SYSTEM" ]; then
         fi
 
         # Unzipping database dump
-        gzip -d $tmpdir/db/$database/$database.*.sql.gz
+        if [ $BACKUP_MODE = 'zstd' ]; then  
+            zstd -d $tmpdir/db/$database/$database.*.sql.zst
+        else
+            gzip -d $tmpdir/db/$database/$database.*.sql.gz
+        fi
 
         # Importing database dump
         database_dump="$tmpdir/db/$database/$database.$TYPE.sql"
@@ -720,9 +759,15 @@ if [ "$udir" != 'no' ]; then
 
         # Creating user dir restore list
         backup_dirs=$(tar -tf $BACKUP/$backup |grep "^./user_dir")
-        backup_dirs=$(echo "$backup_dirs" |grep tar.gz)
-        backup_dirs=$(echo "$backup_dirs" |cut -f 3 -d /)
-        backup_dirs=$(echo "$backup_dirs" |sed "s/.tar.gz//")
+        if [ $BACKUP_MODE = 'zstd' ]; then 
+            backup_dirs=$(echo "$backup_dirs" |grep tar.zst)
+            backup_dirs=$(echo "$backup_dirs" |cut -f 3 -d /)
+            backup_dirs=$(echo "$backup_dirs" |sed "s/.tar.zst//")
+        else
+            backup_dirs=$(echo "$backup_dirs" |grep tar.gz)        
+            backup_dirs=$(echo "$backup_dirs" |cut -f 3 -d /)
+            backup_dirs=$(echo "$backup_dirs" |sed "s/.tar.gz//")
+        fi
         if [ -z "$udir" ] || [ "$udir" = '*' ]; then
             user_dirs="$backup_dirs"
         else
@@ -732,7 +777,11 @@ if [ "$udir" != 'no' ]; then
 
         for user_dir in $user_dirs; do
             echo -e "$(date "+%F %T") $user_dir" |tee -a $tmpdir/restore.log
-            tar xf "$BACKUP/$backup" -C "$tmpdir" --no-wildcards "./user_dir/$user_dir.tar.gz"
+            if [ $BACKUP_MODE = 'zstd' ]; then 
+                tar xf "$BACKUP/$backup" -C "$tmpdir" --no-wildcards "./user_dir/$user_dir.tar.zst"
+            else
+                tar xf "$BACKUP/$backup" -C "$tmpdir" --no-wildcards "./user_dir/$user_dir.tar.gz"
+            fi
             if [ "$?" -ne 0 ]; then
                 rm -rf $tmpdir
                 error="Can't unpack $user_dir user dir container"
@@ -744,7 +793,11 @@ if [ "$udir" != 'no' ]; then
             chown "$user" "$tmpdir/user_dir"
             chown "$user" "$HOMEDIR/$user"
             [ -e "$HOMEDIR/$user/$user_dir" ] && chown "$user" "$HOMEDIR/$user/$user_dir"
-            $BIN/v-extract-fs-archive "$user" "$tmpdir/user_dir/$user_dir.tar.gz" "$HOMEDIR/$user"
+            if [ $BACKUP_MODE = 'zstd' ]; then
+                $BIN/v-extract-fs-archive "$user" "$tmpdir/user_dir/$user_dir.tar.zst" "$HOMEDIR/$user"
+            else
+                $BIN/v-extract-fs-archive "$user" "$tmpdir/user_dir/$user_dir.tar.gz" "$HOMEDIR/$user"
+            fi
             cmdstatus="$?"
             chown root:root "$HOMEDIR/$user"
             if [ "$cmdstatus" -ne 0 ]; then
diff --git a/install/hst-install-debian.sh b/install/hst-install-debian.sh
@@ -44,7 +44,7 @@ if [ "$release" -eq 9 ]; then
         sudo bc ftp lsof rrdtool quota e2fslibs bsdutils e2fsprogs curl
         imagemagick fail2ban dnsutils bsdmainutils cron hestia=${HESTIA_INSTALL_VER} hestia-nginx
         hestia-php expect libmail-dkim-perl unrar-free vim-common acl sysstat
-        rsyslog openssh-server ssh setpriv ipset libapache2-mod-ruid2"
+        rsyslog openssh-server ssh setpriv ipset libapache2-mod-ruid2 zstd"
 elif [ "$release" -eq 10 ]; then
     software="nginx apache2 apache2-utils apache2-suexec-custom
         apache2-suexec-pristine libapache2-mod-fcgid libapache2-mod-php$fpm_v
@@ -61,7 +61,7 @@ elif [ "$release" -eq 10 ]; then
         quota e2fslibs bsdutils e2fsprogs curl imagemagick fail2ban dnsutils
         bsdmainutils cron hestia hestia-nginx hestia-php expect
         libmail-dkim-perl unrar-free vim-common acl sysstat rsyslog openssh-server
-        ssh util-linux ipset libapache2-mpm-itk"
+        ssh util-linux ipset libapache2-mpm-itk zstd"
 fi
 
 installer_dependencies="apt-transport-https curl dirmngr gnupg wget ca-certificates"
@@ -1075,6 +1075,7 @@ fi
 # Backups
 echo "BACKUP_SYSTEM='local'" >> $HESTIA/conf/hestia.conf
 echo "BACKUP_GZIP='9'" >> $HESTIA/conf/hestia.conf
+echo "BACKUP_MODE='zstd'" >> $HESTIA/conf/hestia.conf
 
 # Language
 echo "LANGUAGE='$lang'" >> $HESTIA/conf/hestia.conf
diff --git a/install/hst-install-ubuntu.sh b/install/hst-install-ubuntu.sh
@@ -44,7 +44,7 @@ software="apache2 apache2.2-common apache2-suexec-custom apache2-utils
     postgresql postgresql-contrib proftpd-basic quota roundcube-core
     roundcube-mysql roundcube-plugins rrdtool rssh spamassassin sudo hestia=${HESTIA_INSTALL_VER}
     hestia-nginx hestia-php vim-common vsftpd whois zip acl sysstat setpriv
-    ipset libonig5 libzip5 openssh-server ssh"
+    ipset libonig5 libzip5 openssh-server ssh zstd"
 
 installer_dependencies="apt-transport-https curl dirmngr gnupg wget software-properties-common ca-certificates"
 
@@ -1113,6 +1113,7 @@ fi
 # Backups
 echo "BACKUP_SYSTEM='local'" >> $HESTIA/conf/hestia.conf
 echo "BACKUP_GZIP='9'" >> $HESTIA/conf/hestia.conf
+echo "BACKUP_MODE='zstd'" >> $HESTIA/conf/hestia.conf
 
 # Language
 echo "LANGUAGE='$lang'" >> $HESTIA/conf/hestia.conf
diff --git a/install/upgrade/versions/1.3.0.sh b/install/upgrade/versions/1.3.0.sh
@@ -18,7 +18,7 @@ done
 
 # Add default SSL Certificate config when ip is visited
 if [ "$PROXY_SYSTEM" = "nginx" ]; then
-    echo "[ ! ] Update IP.conf"
+    echo "[ * ] Update IP.conf"
     while read IP; do
         rm /etc/nginx/conf.d/$IP.conf
         cat $WEBTPL/$PROXY_SYSTEM/proxy_ip.tpl |\
@@ -45,11 +45,15 @@ fi
 
 # Remove old lanugage files.
 if [ -e $HESTIA/web/inc/i18n/en.php ]; then 
-    echo "[!] Clean up old language files"
+    echo "[ * ] Clean up old language files"
     rm -fr $HESTIA/web/inc/i18n
 fi
 
 if [ -e "/etc/exim4/exim4.conf.template" ]; then
     echo "[ * ] Updating exim4 configuration..."
     sed -i 's/${if match {${lc:$mime_filename}}{\\N(\\.ade|\\.adp|\\.bat|\\.chm|\\.cmd|\\.com|\\.cpl|\\.exe|\\.hta|\\.ins|\\.isp|\\.jse|\\.lib|\\.lnk|\\.mde|\\.msc|\\.msp|\\.mst|\\.pif|\\.scr|\\.sct|\\.shb|\\.sys|\\.vb|\\.vbe|\\.vbs|\\.vxd|\\.wsc|\\.wsf|\\.wsh)$\\N}{1}{0}}/${if match {${lc:$mime_filename}}{\\N(\\.ace|\\.ade|\\.adp|\\.app|\\.arj|\\.asp|\\.aspx|\\.asx|\\.bas|\\.bat|\\.cab|\\.cer|\\.chm|\\.cmd|\\.cnt|\\.com|\\.cpl|\\.crt|\\.csh|\\.der|\\.diagcab|\\.dll|\\.efi|\\.exe|\\.fla|\\.fon|\\.fxp|\\.gadget|\\.grp|\\.hlp|\\.hpj|\\.hta|\\.htc|\\.img|\\.inf|\\.ins|\\.iso|\\.isp|\\.its|\\.jar|\\.jnlp|\\.js|\\.jse|\\.ksh|\\.lib|\\.lnk|\\.mad|\\.maf|\\.mag|\\.mam|\\.maq|\\.mar|\\.mas|\\.mat|\\.mau|\\.mav|\\.maw|\\.mcf|\\.mda|\\.mdb|\\.mde|\\.mdt|\\.mdw|\\.mdz|\\.msc|\\.msh|\\.msh1|\\.msh1xml|\\.msh2|\\.msh2xml|\\.mshxml|\\.msi|\\.msp|\\.mst|\\.msu|\\.ops|\\.osd|\\.pcd|\\.pif|\\.pl|\\.plg|\\.prf|\\.prg|\\.printerexport|\\.ps1|\\.ps1xml|\\.ps2|\\.ps2xml|\\.psc1|\\.psc2|\\.psd1|\\.psdm1|\\.pst|\\.py|\\.pyc|\\.pyo|\\.pyw|\\.pyz|\\.pyzw|\\.reg|\\.scf|\\.scr|\\.sct|\\.sfx|\\.shb|\\.shs|\\.swf|\\.sys|\\.theme|\\.tmp|\\.ttf|\\.url|\\.vb|\\.vba|\\.vbe|\\.vbp|\\.vbs|\\.vhd|\\.vhdx|\\.vsmacros|\\.vsw|\\.vxd|\\.webpnp|\\.website|\\.wim|\\.ws|\\.wsc|\\.wsf|\\.wsh|\\.xbap|\\.xll|\\.xnk)$\\N}{1}{0}}/g' /etc/exim4/exim4.conf.template
 fi
+
+# Change backup mode to zstd.
+ echo "[ * ] Enable new backup compression zstd as default."
+ $HESTIA/v-change-sys-config-value "BACKUP_MODE" "zstd"
diff --git a/src/deb/hestia/control b/src/deb/hestia/control
@@ -6,7 +6,7 @@ Section: admin
 Maintainer: HestiaCP <info@hestiacp.com>
 Homepage: https://www.hestiacp.com
 Architecture: amd64
-Depends: bash, awk, sed, acl, sysstat, setpriv | util-linux (>= 2.33)
+Depends: bash, awk, sed, acl, sysstat, setpriv | util-linux (>= 2.33), zstd
 Description: hestia
  hestia is an open source hosting control panel.
  hestia has a clean and focused interface without the clutter.
diff --git a/web/edit/server/index.php b/web/edit/server/index.php
@@ -412,6 +412,17 @@
         }
     }
 
+    // Change backup gzip level
+    if (empty($_SESSION['error_msg'])) {
+        if ($_POST['v_backup_mode'] != $v_backup_gzip ) {
+            exec (HESTIA_CMD."v-change-sys-config-value BACKUP_MODE ".escapeshellarg($_POST['v_backup_mode']), $output, $return_var);
+            check_return_code($return_var,$output);
+            unset($output);
+            if (empty($_SESSION['error_msg'])) $v_backup_gzip = $_POST['v_backup_mode'];
+            $v_backup_adv = 'yes';
+        }
+    }
+
     // Change backup path
     if (empty($_SESSION['error_msg'])) {
         if ($_POST['v_backup_dir'] != $v_backup_dir ) {
diff --git a/web/templates/admin/edit_server.html b/web/templates/admin/edit_server.html
