Skip to content

Commit d6a7227

Browse files
serghey-rodinserghey-rodin
committed
Backup exclusions
1 parent 972ecfa commit d6a7227

File tree

3 files changed

+287
-60
lines changed

3 files changed

+287
-60
lines changed

‎bin/v-backup-user‎

Lines changed: 111 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,12 @@ send_mail="$VESTA/web/inc/mail-wrapper.php"
6262
la=$(cat /proc/loadavg | cut -f 1 -d ' ' | cut -f 1 -d '.')
6363
i=0
6464
while [ "$la" -ge "$BACKUP_LA_LIMIT" ]; do
65-
echo "$(date "+%F %T") Load Average $la"
66-
echo
67-
sleep 60
68-
if [ "$i" -ge "15" ]; then
69-
echo "LoadAverage $i is above threshold" | $send_mail -s "$subj" $email
65+
echo "$(date "+%F %T") LoadAverage $la is above threshold. Sleeping..."
66+
sleep 120
67+
if [ "$i" -ge "5" ]; then
68+
mail_top=$(top -b| head -n 30)
69+
mail_text="LoadAverage $i is above threshold\n\n$mail_top\n"
70+
echo -e "$mail_text" | $send_mail -s "$subj" $email
7071
echo "Error: LA is too high"
7172
sed -i "/ $user /d" $VESTA/data/queue/backup.pipe
7273
log_event "$E_LA" "$EVENT"
@@ -106,10 +107,10 @@ if [ -e "$USER_DATA/history.log" ]; then
106107
cp -r $USER_DATA/history.log $tmpdir/vesta/
107108
fi
108109

109-
if [ -e "$USER_DATA/backup.excludes" ]; then
110-
echo -e "$(date "+%F %T") backup.excludes"
111-
msg="$msg\n$(date "+%F %T") backup.excludes"
112-
cp -r $USER_DATA/backup.excludes $tmpdir/vesta/
110+
if [ -e "$USER_DATA/backup-excludes.conf" ]; then
111+
echo -e "$(date "+%F %T") backup-excludes.conf"
112+
msg="$msg\n$(date "+%F %T") backup-excludes.conf"
113+
cp -r $USER_DATA/backup-excludes.conf $tmpdir/vesta/
113114
fi
114115

115116
# Backup PAM
@@ -123,27 +124,25 @@ echo
123124
msg="$msg\n"
124125

125126
# Parsing excludes
126-
OLD_IFS="$IFS"
127-
IFS=$'\n'
128-
if [ -e "$USER_DATA/backup.excludes" ]; then
129-
for exclude in $(cat $USER_DATA/backup.excludes); do
130-
eval ${exclude%%=*}=${exclude#*=}
131-
done
127+
if [ -e "$USER_DATA/backup-excludes.conf" ]; then
128+
source $USER_DATA/backup-excludes.conf
132129
fi
133-
IFS="$OLD_IFS"
134130

135131
# WEB domains
136132
if [ ! -z "$WEB_SYSTEM" ] && [ "$WEB" != '*' ]; then
137133
echo "-- WEB --"
138134
msg="$msg\n-- WEB --"
139135
mkdir $tmpdir/web/
140136

141-
# Parsing unsuspeneded domains
137+
# Parsing web domain exclusions
142138
conf="$USER_DATA/web.conf"
143139
for domain in $(search_objects 'web' 'SUSPENDED' "*" 'DOMAIN'); do
144-
check_exl=$(echo "$WEB"|grep -w $domain)
140+
check_exl=$(echo -e "${WEB//,/\n}" |grep "^$domain$")
145141
if [ -z "$check_exl" ]; then
146142
web_list="$web_list $domain"
143+
else
144+
echo "$(date "+%F %T") excluding $domain"
145+
msg="$msg\n$(date "+%F %T") excluding $domain"
147146
fi
148147
done
149148
web_list=$(echo "$web_list" | sed -e "s/ */\ /g" -e "s/^ //")
@@ -207,9 +206,26 @@ if [ ! -z "$WEB_SYSTEM" ] && [ "$WEB" != '*' ]; then
207206
fi
208207

209208
# Packing data folders
209+
touch $tmpdir/web/$domain/domain_data.tar
210210
cd $HOMEDIR/$user/web/$domain
211-
domain_direcotries=$(ls |grep -v logs)
212-
tar -cpf $tmpdir/web/$domain/domain_data.tar $domain_direcotries
211+
set -f
212+
fargs+=(-not)
213+
fargs+=(-path)
214+
fargs+=("./logs*")
215+
check_exlusion=$(echo -e "${WEB//,/\n}" | grep "^$domain:")
216+
if [ ! -z "$check_exlusion" ]; then
217+
xdirs="$(echo -e "${check_exlusion//:/\n}" |grep -v $domain)"
218+
for xpath in $xdirs; do
219+
xpath="$(echo $xpath | sed -e 's/\/*$//' -e 's/^\/*//')"
220+
fargs+=(-not)
221+
fargs+=(-path)
222+
fargs+=("./$xpath*")
223+
echo "$(date "+%F %T") excluding directory $xpath"
224+
msg="$msg\n$(date "+%F %T") excluding directory $xpath"
225+
done
226+
fi
227+
find . ${fargs[@]} |grep -v "^./$" | grep -v "^.$" |\
228+
xargs tar -rpf $tmpdir/web/$domain/domain_data.tar
213229
gzip -$BACKUP_GZIP $tmpdir/web/$domain/domain_data.tar
214230
done
215231

@@ -230,11 +246,14 @@ if [ ! -z "$DNS_SYSTEM" ] && [ "$DNS" != '*' ]; then
230246
msg="$msg\n-- DNS --"
231247
mkdir $tmpdir/dns/
232248

233-
# Parsing unsuspeneded domains
249+
# Parsing dns domain exclusions
234250
for domain in $(search_objects 'dns' 'SUSPENDED' "*" 'DOMAIN'); do
235-
check_exl=$(echo "$DNS"|grep -w $domain)
251+
check_exl=$(echo -e "${DNS//,/\n}" |grep "^$domain$")
236252
if [ -z "$check_exl" ]; then
237253
dns_list="$dns_list $domain"
254+
else
255+
echo "$(date "+%F %T") excluding $domain"
256+
msg="$msg\n$(date "+%F %T") excluding $domain"
238257
fi
239258
done
240259
dns_list=$(echo "$dns_list" | sed -e "s/ */\ /g" -e "s/^ //")
@@ -256,7 +275,9 @@ if [ ! -z "$DNS_SYSTEM" ] && [ "$DNS" != '*' ]; then
256275

257276
# Backingup dns recods
258277
cp $USER_DATA/dns/$domain.conf vesta/$domain.conf
259-
cp $HOMEDIR/$user/conf/dns/$domain.db conf/$domain.db
278+
if [ "$DNS_SYSTEM" != 'remote' ]; then
279+
cp $HOMEDIR/$user/conf/dns/$domain.db conf/$domain.db
280+
fi
260281
done
261282

262283
if [ "$i" -eq 1 ]; then
@@ -276,12 +297,15 @@ if [ ! -z "$MAIL_SYSTEM" ] && [ "$MAIL" != '*' ]; then
276297
msg="$msg\n-- MAIL --"
277298
mkdir $tmpdir/mail/
278299

279-
# Parsing unsuspeneded domains
300+
# Parsing mail domain exclusions
280301
conf="$USER_DATA/mail.conf"
281302
for domain in $(search_objects 'mail' 'SUSPENDED' "*" 'DOMAIN'); do
282-
check_exl=$(echo "$MAIL"|grep -w $domain)
303+
check_exl=$(echo -e "${MAIL//,/\n}" |grep "^$domain$")
283304
if [ -z "$check_exl" ]; then
284305
mail_list="$mail_list $domain"
306+
else
307+
echo "$(date "+%F %T") excluding $domain"
308+
msg="$msg\n$(date "+%F %T") excluding $domain"
285309
fi
286310
done
287311
mail_list=$(echo "$mail_list" | sed -e "s/ */\ /g" -e "s/^ //")
@@ -311,9 +335,19 @@ if [ ! -z "$MAIL_SYSTEM" ] && [ "$MAIL" != '*' ]; then
311335

312336
# Packing mailboxes
313337
cd $HOMEDIR/$user/mail/$domain_idn
314-
accounts=$(ls)
315-
if [ ! -z "$accounts" ] && [[ "$MAIL_SYSTEM" =~ exim ]]; then
316-
tar -cpf $tmpdir/mail/$domain/accounts.tar $accounts
338+
for account in $(ls); do
339+
exclusion=$(echo -e "${MAIL//,/\n}" |grep "$domain:")
340+
exclusion=$(echo -e "${exclusion//:/\n}" |grep "^$account$")
341+
if [ -z "$exclusion" ] && [[ "$MAIL_SYSTEM" =~ exim ]]; then
342+
echo "$(date "+%F %T") $account"
343+
touch $tmpdir/mail/$domain/accounts.tar
344+
tar -rpf $tmpdir/mail/$domain/accounts.tar $account
345+
else
346+
echo "$(date "+%F %T") excluding account $account"
347+
msg="$msg\n$(date "+%F %T") excluding account $account"
348+
fi
349+
done
350+
if [ -e "$tmpdir/mail/$domain/accounts.tar" ]; then
317351
gzip -$BACKUP_GZIP $tmpdir/mail/$domain/accounts.tar
318352
fi
319353
done
@@ -336,11 +370,14 @@ if [ ! -z "$DB_SYSTEM" ] && [ "$DB" != '*' ]; then
336370
msg="$msg\n-- DB --"
337371
mkdir $tmpdir/db/
338372

339-
# Parsing unsuspeneded domains
373+
# Parsing database exclusions
340374
for database in $(search_objects 'db' 'SUSPENDED' "*" 'DB'); do
341-
check_exl=$(echo "$DB"|grep -w $database)
375+
check_exl=$(echo -e "${DB//,/\n}" |grep "^$database$")
342376
if [ -z "$check_exl" ]; then
343377
db_list="$db_list $database"
378+
else
379+
echo "$(date "+%F %T") excluding $database"
380+
msg="$msg\n$(date "+%F %T") excluding $database"
344381
fi
345382
done
346383
db_list=$(echo "$db_list" | sed -e "s/ */\ /g" -e "s/^ //")
@@ -410,38 +447,52 @@ if [ ! -z "$CRON_SYSTEM" ] && [ "$CRON" != '*' ]; then
410447
fi
411448

412449
# User Directories
413-
echo "-- User Dir --"
414-
msg="$msg\n-- User Dir --"
415-
mkdir $tmpdir/user_dir
416-
user_dir=$(ls $HOMEDIR/$user | \
417-
grep -v conf | \
418-
grep -v web | \
419-
grep -v dns | \
420-
grep -v mail | \
421-
grep -v tmp)
422-
i=0
423-
for udir in $user_dir; do
424-
udir_list="$udir_list $udir"
425-
((i ++))
426-
echo -e "$(date "+%F %T") $udir"
427-
msg="$msg\n$(date "+%F %T") $udir"
428-
cp -pr $HOMEDIR/$user/$udir $tmpdir/user_dir/
429-
cd $tmpdir/user_dir/
430-
tar -czpf $udir.tar.gz $udir
431-
rm -rf $udir
432-
done
433-
udir_list=$(echo "$udir_list" | sed -e "s/ */\ /g" -e "s/^ //")
450+
if [ "$USER" != '*' ]; then
451+
echo "-- User Dir --"
452+
msg="$msg\n-- User Dir --"
453+
mkdir $tmpdir/user_dir
454+
cd $HOMEDIR/$user
455+
456+
# Default excludes
457+
set -f
458+
fargs=''
459+
460+
# Parsing directory exlusion list
461+
exlusion_list=$(echo -e "${USER//,/\n}")
462+
for xpath in $exlusion_list; do
463+
fargs+=(-not)
464+
fargs+=(-path)
465+
fargs+=("./$xpath*")
466+
echo "$(date "+%F %T") excluding directory $xpath"
467+
msg="$msg\n$(date "+%F %T") excluding directory $xpath"
468+
done
434469

435-
if [ "$i" -eq 1 ]; then
436-
echo -e "$(date "+%F %T") $i user directory"
437-
msg="$msg\n$(date "+%F %T") $i directory"
438-
else
439-
echo -e "$(date "+%F %T") $i directories"
440-
msg="$msg\n$(date "+%F %T") $i directories"
441-
fi
442-
echo
443-
msg="$msg\n"
470+
for udir in $(ls |egrep -v "conf|web|dns|mail"); do
471+
check_exl=$(echo -e "${USER//,/\n}" |grep "^$udir$")
472+
if [ -z "$check_exl" ]; then
473+
((i ++))
474+
udir_list="$udir_list $udir"
475+
echo -e "$(date "+%F %T") adding directory $udir"
476+
msg="$msg\n$(date "+%F %T") adding directory $udir"
477+
touch $tmpdir/user_dir/$udir.tar
478+
find ./$udir ${fargs[@]} |grep -v "^./$" |grep -v "^.$" |\
479+
grep -v "./$udir$" |\
480+
xargs tar -rpf $tmpdir/user_dir/$udir.tar
481+
gzip -$BACKUP_GZIP $tmpdir/user_dir/$udir.tar
482+
fi
483+
done
484+
udir_list=$(echo "$udir_list" | sed -e "s/ */\ /g" -e "s/^ //")
444485

486+
if [ "$i" -eq 1 ]; then
487+
echo -e "$(date "+%F %T") $i user directory"
488+
msg="$msg\n$(date "+%F %T") $i directory"
489+
else
490+
echo -e "$(date "+%F %T") $i directories"
491+
msg="$msg\n$(date "+%F %T") $i directories"
492+
fi
493+
echo
494+
msg="$msg\n"
495+
fi
445496

446497
# Get backup size
447498
size="$(du -shm $tmpdir | cut -f 1)"

‎bin/v-list-user-backup-exclusions‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/bin/bash
2+
# info: list backup exclusions
3+
# options: USER [FORMAT]
4+
#
5+
# The function for obtaining the backup exclusion list
6+
7+
8+
#----------------------------------------------------------#
9+
# Variable&Function #
10+
#----------------------------------------------------------#
11+
12+
# Argument defenition
13+
user=$1
14+
format=${2-shell}
15+
USER=''
16+
17+
# Includes
18+
source $VESTA/func/main.sh
19+
20+
# Json function
21+
json_list_backup_xld() {
22+
set -f
23+
echo '{'
24+
fields_count=$(echo "$fields" | wc -w)
25+
i=1
26+
source $USER_DATA/backup-excludes.conf
27+
for field in $fields; do
28+
eval value=$field
29+
j=1
30+
echo -e "\t\"${field//$/}\": {"
31+
for exlcude in ${value//,/ }; do
32+
exlcude_obj=$(echo $exlcude |cut -f 1 -d:)
33+
exclude_param=$(echo $exlcude |sed -e "s/$exlcude_obj://")
34+
if [ "$exlcude_obj" = "$exclude_param" ]; then
35+
exclude_param=''
36+
fi
37+
if [ $j -lt "$(echo ${value//,/ } |wc -w)" ]; then
38+
echo -e "\t\t\"$exlcude_obj\": \"$exclude_param\","
39+
else
40+
echo -e "\t\t\"$exlcude_obj\": \"$exclude_param\""
41+
fi
42+
(( ++j))
43+
done
44+
if [ $i -lt $fields_count ]; then
45+
echo -e "\t},"
46+
else
47+
echo -e "\t}"
48+
fi
49+
(( ++i))
50+
done
51+
echo '}'
52+
}
53+
54+
# Shell function
55+
shell_list_backup_xld() {
56+
source $USER_DATA/backup-excludes.conf
57+
for field in $fields; do
58+
eval value=$field
59+
echo -e "${field//$/}: $value"
60+
done
61+
}
62+
63+
64+
#----------------------------------------------------------#
65+
# Verifications #
66+
#----------------------------------------------------------#
67+
68+
check_args '1' "$#" 'USER [FORMAT]'
69+
validate_format 'user'
70+
is_object_valid 'user' 'USER' "$user"
71+
72+
73+
#----------------------------------------------------------#
74+
# Action #
75+
#----------------------------------------------------------#
76+
77+
# Defining fileds to select
78+
conf=$VESTA/data/ips/*
79+
fields="\$WEB \$DNS \$MAIL \$DB \$CRON \$USER"
80+
81+
# Listing backup exclusions
82+
case $format in
83+
json) json_list_backup_xld ;;
84+
plain) nohead=1; shell_list_backup_xld ;;
85+
shell) shell_list_backup_xld;;
86+
*) check_args '1' '0' '[FORMAT]'
87+
esac
88+
89+
90+
#----------------------------------------------------------#
91+
# Vesta #
92+
#----------------------------------------------------------#
93+
94+
exit

0 commit comments

Comments
 (0)