Make ZSTD multi-threaded (hestiacp#1593)

Alexandros Ioannides · jaapmarcus · web-flow · commit fd0a96b545f7 · 2021-01-28T07:43:44.000+01:00
* Multi-threaded compression for zstd

* Update CHANGELOG.md

* Multi threaded zstd

* Multi-threaded decompression for ZSTD

* Added 10 - 19 “Compression” level
Added link to docs.

* Set “Default” for new users to 4

Co-authored-by: Jaap Marcus &lt;9754650+jaapmarcus@users.noreply.github.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file.
 - Fixed an issue where local and FTP backup files were not deleted when running `v-delete-user-backup`. (#1421)
 - Fixed an issue where IP addresses could not be deleted (#1423)
 - Improvements have been made to the API's error handling - thanks **@danielalexis**!
+- ZSTD Compression has been made multi-threaded.
 
 ## [1.3.2] - Service Release
 ### Features
diff --git a/bin/v-backup-user b/bin/v-backup-user
@@ -271,7 +271,7 @@ if [ ! -z "$WEB_SYSTEM" ] && [ "$WEB" != '*' ]; then
 
         # Backup files
         if [ "$BACKUP_MODE" = 'zstd' ]; then
-            tar ${fargs[@]} -cpf- * | zstd -$BACKUP_GZIP - > $tmpdir/web/$domain/domain_data.tar.zst
+            tar ${fargs[@]} -cpf- * | pzstd -$BACKUP_GZIP - > $tmpdir/web/$domain/domain_data.tar.zst
         else    
             tar ${fargs[@]} -cpf- * | gzip -$BACKUP_GZIP - > $tmpdir/web/$domain/domain_data.tar.gz
         fi
@@ -390,7 +390,7 @@ if [ ! -z "$MAIL_SYSTEM" ] && [ "$MAIL" != '*' ]; then
         # Compress archive
         if [ ${#accounts[@]} -gt 0 ]; then
             if [ "$BACKUP_MODE" = 'zstd' ]; then
-                tar -cpf- ${accounts[@]}  | zstd -$BACKUP_GZIP - > $tmpdir/mail/$domain/accounts.tar.zst
+                tar -cpf- ${accounts[@]}  | pzstd -$BACKUP_GZIP - > $tmpdir/mail/$domain/accounts.tar.zst
             else    
                 tar -cpf- ${accounts[@]} | gzip -$BACKUP_GZIP - > $tmpdir/mail/$domain/accounts.tar.gz
             fi        
@@ -470,7 +470,7 @@ if [ ! -z "$DB_SYSTEM" ] && [ "$DB" != '*' ]; then
 
             # Compress dump
             if [ "$BACKUP_MODE" = 'zstd' ]; then
-                zstd -$BACKUP_GZIP $dump
+                pzstd -$BACKUP_GZIP $dump
                 rm $dump;
             else    
                 gzip -$BACKUP_GZIP $dump
@@ -551,7 +551,7 @@ if [ "$USER" != '*' ]; then
 
             # Backup files and dirs
             if [ "$BACKUP_MODE" = 'zstd' ]; then
-                tar --anchored -cpf- ${fargs[@]} $udir | zstd -$BACKUP_GZIP - > $tmpdir/user_dir/$udir.tar.zst
+                tar --anchored -cpf- ${fargs[@]} $udir | pzstd -$BACKUP_GZIP - > $tmpdir/user_dir/$udir.tar.zst
             else    
                 tar --anchored -cpf- ${fargs[@]} $udir | gzip -$BACKUP_GZIP - > $tmpdir/user_dir/$udir.tar.gz
             fi 
diff --git a/bin/v-extract-fs-archive b/bin/v-extract-fs-archive
@@ -90,10 +90,10 @@ if [ ! -z "$(echo $src_file |egrep -i  '.tar.zst')" ]; then
     x='yes'
     if [ -z "$test" ] || [ "$test" = "no" ]; then
         user_exec mkdir -p "$dst_dir" >/dev/null 2>&1
-        user_exec tar -I zstd -xf "$src_file" -C "$dst_dir" --no-wildcards "$selected_dir" $tar_strip_level >/dev/null 2>&1
+        user_exec tar -I pzstd -xf "$src_file" -C "$dst_dir" --no-wildcards "$selected_dir" $tar_strip_level >/dev/null 2>&1
         rc=$?
     else
-        user_exec tar -I zstd -tf "$src_file" --no-wildcards "$selected_dir" >/dev/null 2>&1
+        user_exec tar -I pzstd -tf "$src_file" --no-wildcards "$selected_dir" >/dev/null 2>&1
         rc=$?
     fi
 fi
@@ -102,7 +102,7 @@ fi
 if [ ! -z "$(echo $src_file |grep -i  '.zst')" ] && [ -z "$x" ]; then
     user_exec mkdir -p "$dst_dir" >/dev/null 2>&1
     user_exec mv "$src_file" "$dst_dir" >/dev/null 2>&1
-    user_exec zstd -d "$dst_dir/$(basename $src_file)" >/dev/null 2>&1
+    user_exec pzstd -d "$dst_dir/$(basename $src_file)" >/dev/null 2>&1
     rc=$?
 fi
 
diff --git a/bin/v-restore-user b/bin/v-restore-user
@@ -330,7 +330,7 @@ if [ "$web" != 'no' ] && [ ! -z "$WEB_SYSTEM" ]; then
         [[ -d $HOMEDIR/$user/web/$domain/stats ]] && chmod u+w "$HOMEDIR/$user/web/$domain/stats"
     
         if [ "$backup_mode" = "zstd" ]; then
-            user_exec tar -I zstd -xpf $tmpdir/web/$domain/domain_data.tar.zst \
+            user_exec tar -I pzstd -xpf $tmpdir/web/$domain/domain_data.tar.zst \
                 -C "$HOMEDIR/$user/web/$domain/" \
                 --anchored \
                 --exclude='logs/*'
@@ -709,7 +709,7 @@ if [ "$db" != 'no' ] && [ ! -z "$DB_SYSTEM" ]; then
 
         # Unzipping database dump
         if [ $backup_mode = 'zstd' ]; then  
-            zstd -d $tmpdir/db/$database/$database.*.sql.zst
+            pzstd -d $tmpdir/db/$database/$database.*.sql.zst
         else
             gzip -d $tmpdir/db/$database/$database.*.sql.gz
         fi
diff --git a/install/hst-install-debian.sh b/install/hst-install-debian.sh
@@ -1076,7 +1076,7 @@ fi
 
 # Backups
 echo "BACKUP_SYSTEM='local'" >> $HESTIA/conf/hestia.conf
-echo "BACKUP_GZIP='9'" >> $HESTIA/conf/hestia.conf
+echo "BACKUP_GZIP='4'" >> $HESTIA/conf/hestia.conf
 echo "BACKUP_MODE='zstd'" >> $HESTIA/conf/hestia.conf
 
 # Language
diff --git a/install/hst-install-ubuntu.sh b/install/hst-install-ubuntu.sh
@@ -1126,7 +1126,7 @@ fi
 
 # Backups
 echo "BACKUP_SYSTEM='local'" >> $HESTIA/conf/hestia.conf
-echo "BACKUP_GZIP='9'" >> $HESTIA/conf/hestia.conf
+echo "BACKUP_GZIP='4'" >> $HESTIA/conf/hestia.conf
 echo "BACKUP_MODE='zstd'" >> $HESTIA/conf/hestia.conf
 
 # Language
diff --git a/web/edit/server/index.php b/web/edit/server/index.php
@@ -437,6 +437,9 @@
     // Change backup gzip level
     if (empty($_SESSION['error_msg'])) {
         if ($_POST['v_backup_gzip'] != $v_backup_gzip ) {
+            if ( $_POST['v_backup_mode'] == 'gzip' ){
+                $_POST['v_backup_gzip'] = 9;
+            }
             exec (HESTIA_CMD."v-change-sys-config-value BACKUP_GZIP ".escapeshellarg($_POST['v_backup_gzip']), $output, $return_var);
             check_return_code($return_var,$output);
             unset($output);
@@ -445,14 +448,20 @@
         }
     }
 
-    // Change backup gzip level
+    // Change backup mode
     if (empty($_SESSION['error_msg'])) {
         if ($_POST['v_backup_mode'] != $v_backup_mode ) {
             exec (HESTIA_CMD."v-change-sys-config-value BACKUP_MODE ".escapeshellarg($_POST['v_backup_mode']), $output, $return_var);
             check_return_code($return_var,$output);
             unset($output);
             if (empty($_SESSION['error_msg'])) $v_backup_mode = $_POST['v_backup_mode'];
             $v_backup_adv = 'yes';
+            if ( $_POST['v_backup_mode'] == 'gzip' ){
+                $_POST['v_backup_gzip'] = 9;
+                if (empty($_SESSION['error_msg'])) $v_backup_gzip = $_POST['v_backup_gzip'];
+                exec (HESTIA_CMD."v-change-sys-config-value BACKUP_GZIP ".escapeshellarg($_POST['v_backup_gzip']), $output, $return_var);
+            }
+            
         }
     }
 
diff --git a/web/templates/admin/edit_server.html b/web/templates/admin/edit_server.html
@@ -602,8 +602,6 @@
                                     </table>
                                 </td>
                             </tr>
-
-
                             <tr>
                                 <td class="vst-text input-label step-top advanced-options">
                                     <a href="javascript:elementHideShow('backup');" class="vst-text">
@@ -630,7 +628,7 @@
                                         </tr>
                                         <tr>
                                             <td class="vst-text">
-                                                <?php print _('Compression') ?>
+                                                <?php print _('Compression') ?> <a href="https://docs.hestiacp.com/admin_docs/backups.html"><i class="fas fa-question-circle"></i></a>
                                             </td>
                                         </tr>
                                         <tr>
@@ -644,21 +642,18 @@
                                         </tr>
                                         <tr>
                                             <td class="vst-text">
-                                                <?php print _('Compression level') ?>
+                                                <?php print _('Compression level') ?> <a href="https://docs.hestiacp.com/admin_docs/backups.html"><i class="fas fa-question-circle"></i></a>
                                             </td>
                                         </tr>
                                         <tr>
                                             <td>
                                                 <select class="vst-list" name="v_backup_gzip">
-                                                    <option value='1' <?php if($v_backup_gzip == '1') echo 'selected' ?>>1</option>
-                                                    <option value='2' <?php if($v_backup_gzip == '2') echo 'selected' ?>>2</option>
-                                                    <option value='3' <?php if($v_backup_gzip == '3') echo 'selected' ?>>3</option>
-                                                    <option value='4' <?php if($v_backup_gzip == '4') echo 'selected' ?>>4</option>
-                                                    <option value='5' <?php if($v_backup_gzip == '5') echo 'selected' ?>>5</option>
-                                                    <option value='6' <?php if($v_backup_gzip == '6') echo 'selected' ?>>6</option>
-                                                    <option value='7' <?php if($v_backup_gzip == '7') echo 'selected' ?>>7</option>
-                                                    <option value='8' <?php if($v_backup_gzip == '8') echo 'selected' ?>>8</option>
-                                                    <option value='9' <?php if($v_backup_gzip == '9') echo 'selected' ?>>9</option>
+                                                <?php for($level = 1; $level < 20; $level++){
+                                                ?>
+                                                <option value='<?=$level;?>' <?php if($v_backup_gzip == $level) echo 'selected' ?>><?=$level;?><?php if($level > 9){ echo sprintf(' (%s)', _('zstd only')); } ?></option>
+                                                <?php
+                                                }
+                                                ?>
                                                 </select>
                                                 <br><br>
                                             </td>
@@ -673,7 +668,8 @@
                                                 <input type="text" size="20" class="vst-input" name="v_backup_dir" value="<?=trim($v_backup_dir, "'")?>">
                                                 <br><br>
                                             </td>
-                                        </tr>                                        <tr>
+                                        </tr>
+                                        <tr>
                                             <td class="vst-text input-label">
                                                     <label><input type="checkbox" size="20" class="vst-checkbox" name="v_backup_remote_adv" <?php if (!empty($v_backup_remote_adv)) echo "checked=yes" ?> onclick="javascript:elementHideShow('remote_backup');"> <?php print _('Remote backup');?></label>
                                             </td>
@@ -682,7 +678,7 @@
                                             <td class="vst-text input-label step-left">
                                                 <table style="display:<?php if (empty($v_backup_remote_adv)) echo 'none';?> ;" id="remote_backup">
                                                     <td class="vst-text">
-                                                        <?php print _('Protocol') ?>
+                                                        <?php print _('Protocol') ?> <a href="https://docs.hestiacp.com/admin_docs/backups.html"><i class="fas fa-question-circle"></i></a>
                                                     </td>
                                                     </tr>
                                                     <tr>
