Skip to content

Commit a14556d

Browse files
serghey-rodinserghey-rodin
committed
unofficial google nearline support
1 parent 137f258 commit a14556d

File tree

2 files changed

+163
-8
lines changed

2 files changed

+163
-8
lines changed

‎bin/v-backup-user‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -747,6 +747,46 @@ sftp_backup() {
747747
fi
748748
}
749749

750+
google_backup() {
751+
752+
# Defining google settings
753+
source $VESTA/conf/google.backup.conf
754+
gsutil="$VESTA/3rdparty/gsutil/gsutil"
755+
export BOTO_CONFIG="$VESTA/conf/.google.backup.boto"
756+
757+
# Debug info
758+
echo -e "$(date "+%F %T") Remote: gs://$BUCKET/$BPATH/$user.$date.tar"
759+
760+
# Checking retention
761+
backup_list=$(${gsutil} ls gs://$BUCKET/$BPATH/$user.* 2>/dev/null)
762+
backups_count=$(echo "$backup_list" |wc -l)
763+
if [ "$backups_count" -ge "$BACKUPS" ]; then
764+
backups_rm_number=$((backups_count - BACKUPS + 1))
765+
for backup in $(echo "$backup_list" |head -n $backups_rm_number); do
766+
echo -e "$(date "+%F %T") Roated gcp backup: $backup"
767+
$gsutil rm $backup > /dev/null 2>&1
768+
done
769+
fi
770+
771+
# Uploading backup archive
772+
echo -e "$(date "+%F %T") Uploading $user.$date.tar ..."
773+
if [ "$localbackup" = 'yes' ]; then
774+
cd $BACKUP
775+
${gsutil} cp $user.$date.tar gs://$BUCKET/$BPATH/ > /dev/null 2>&1
776+
else
777+
cd $tmpdir
778+
tar -cf $BACKUP/$user.$date.tar .
779+
cd $BACKUP/
780+
${gsutil} cp $user.$date.tar gs://$BUCKET/$BPATH/ > /dev/null 2>&1
781+
rc=$?
782+
rm -f $user.$date.tar
783+
if [ "$rc" -ne 0 ]; then
784+
check_result "$E_CONNECT" "gsutil failed to upload $user.$date.tar"
785+
fi
786+
fi
787+
}
788+
789+
750790
echo -e "\n-- SUMMARY --" |tee -a $BACKUP/$user.log
751791

752792
# Switching on backup system types
@@ -755,6 +795,7 @@ for backup_type in $(echo -e "${BACKUP_SYSTEM//,/\\n}"); do
755795
local) local_backup ;;
756796
ftp) ftp_backup ;;
757797
sftp) sftp_backup ;;
798+
google) google_backup ;;
758799
esac
759800
done
760801

‎bin/v-restore-user‎

Lines changed: 122 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,20 +36,116 @@ source $VESTA/func/db.sh
3636
source $VESTA/func/rebuild.sh
3737
source $VESTA/conf/vesta.conf
3838

39-
# Check backup function
40-
is_backup_valid() {
41-
if [ ! -e "$1" ]; then
42-
check_result $E_NOTEXIST "backup $1 doesn't exist"
43-
fi
44-
}
45-
4639
# Check backup ownership function
4740
is_backup_available() {
4841
if ! [[ $2 =~ ^$1.[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].tar$ ]]; then
4942
check_result $E_FORBIDEN "permission denied"
5043
fi
5144
}
5245

46+
# Defining ftp command function
47+
ftpc() {
48+
/usr/bin/ftp -n $HOST $PORT <<EOF
49+
quote USER $USERNAME
50+
quote PASS $PASSWORD
51+
binary
52+
$1
53+
$2
54+
$3
55+
quit
56+
EOF
57+
}
58+
59+
# FTP backup download function
60+
ftp_download() {
61+
source $VESTA/conf/ftp.backup.conf
62+
if [ -z "$PORT" ]; then
63+
PORT='21'
64+
fi
65+
ftpc "cd $BPATH" "get $1"
66+
}
67+
68+
# sftp command function
69+
sftpc() {
70+
expect -f "-" <<EOF "$@"
71+
set timeout 60
72+
set count 0
73+
spawn /usr/bin/sftp -o StrictHostKeyChecking=no \
74+
-o Port=$PORT $USERNAME@$HOST
75+
expect {
76+
"password:" {
77+
send "$PASSWORD\r"
78+
exp_continue
79+
}
80+
81+
-re "Couldn't|(.*)disconnect|(.*)stalled|(.*)not found" {
82+
set count \$argc
83+
set output "Disconnected."
84+
set rc $E_FTP
85+
exp_continue
86+
}
87+
88+
-re ".*denied.*(publickey|password)." {
89+
set output "Permission denied, wrong publickey or password."
90+
set rc $E_CONNECT
91+
}
92+
93+
-re "\[0-9]*%" {
94+
exp_continue
95+
}
96+
97+
"sftp>" {
98+
if {\$count < \$argc} {
99+
set arg [lindex \$argv \$count]
100+
send "\$arg\r"
101+
incr count
102+
} else {
103+
incr count
104+
} else {
105+
send "exit\r"
106+
set output "Disconnected."
107+
if {[info exists rc] != 1} {
108+
set rc $OK
109+
}
110+
}
111+
exp_continue
112+
}
113+
timeout {
114+
set output "Connection timeout."
115+
set rc $E_CONNECT
116+
}
117+
}
118+
119+
if {[info exists output] == 1} {
120+
puts "\$output"
121+
}
122+
123+
exit \$rc
124+
EOF
125+
}
126+
127+
# SFTP backup download function
128+
sftp_download() {
129+
source $VESTA/conf/sftp.backup.conf
130+
if [ -z "$PORT" ]; then
131+
PORT='22'
132+
fi
133+
cd $BACKUP
134+
sftpc "cd $BPATH" "get $1" > /dev/null 2>&1
135+
136+
}
137+
138+
# Google backup download function
139+
google_download() {
140+
source $VESTA/conf/google.backup.conf
141+
gsutil="$VESTA/3rdparty/gsutil/gsutil"
142+
export BOTO_CONFIG="$VESTA/conf/.google.backup.boto"
143+
${gsutil} cp gs://$BUCKET/$BPATH/$1 $BACKUP/ > /dev/null 2>&1
144+
if [ "$?" -ne 0 ]; then
145+
check_result "$E_CONNECT" "gsutil failed to download $1"
146+
fi
147+
}
148+
53149

54150
#----------------------------------------------------------#
55151
# Verifications #
@@ -58,14 +154,32 @@ is_backup_available() {
58154
args_usage='USER BACKUP [WEB] [DNS] [MAIL] [DB] [CRON] [UDIR] [NOTIFY]'
59155
check_args '2' "$#" "$args_usage"
60156
is_format_valid 'user' 'backup'
61-
is_backup_valid "$BACKUP/$backup"
62157
is_backup_available "$user" "$backup"
63158

64159

65160
#----------------------------------------------------------#
66161
# Action #
67162
#----------------------------------------------------------#
68163

164+
# Checking local backup
165+
if [ ! -e "$BACKUP/$backup" ]; then
166+
if [[ "$BACKUP_SYSTEM" =~ "google" ]]; then
167+
google_download $backup
168+
downloaded='yes'
169+
fi
170+
if [[ "$BACKUP_SYSTEM" =~ "sftp" ]] && [ -z "$downloaded" ]; then
171+
sftp_download $backup
172+
downloaded='yes'
173+
fi
174+
if [[ "$BACKUP_SYSTEM" =~ "ftp" ]] && [ -z "$downloaded" ]; then
175+
ftp_download $backup
176+
downloaded='yes'
177+
fi
178+
if [ -z "$downloaded" ]; then
179+
check_result $E_NOTEXIST "backup $backup doesn't exist"
180+
fi
181+
fi
182+
69183
# Checking user existance on the server
70184
check_user=$(is_object_valid 'user' 'USER' "$user")
71185
if [ -z "$check_user" ]; then

0 commit comments

Comments
 (0)