Skip to content

Commit 85af073

Browse files
matthewpimatthewpi
committed
Switch to s3 multipart uploads for backups
1 parent 23d2352 commit 85af073

File tree

7 files changed

+137
-41
lines changed

7 files changed

+137
-41
lines changed

‎app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
<?php
2+
3+
namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
4+
5+
use Carbon\CarbonImmutable;
6+
use Illuminate\Http\Request;
7+
use Illuminate\Http\JsonResponse;
8+
use League\Flysystem\AwsS3v3\AwsS3Adapter;
9+
use Pterodactyl\Http\Controllers\Controller;
10+
use Pterodactyl\Extensions\Backups\BackupManager;
11+
use Pterodactyl\Repositories\Eloquent\BackupRepository;
12+
13+
class BackupRemoteUploadController extends Controller
14+
{
15+
// I would use 1024 but I'm unsure if AWS or other S3 servers,
16+
// use SI gigabyte (base 10), or the proper IEC gibibyte (base 2).
17+
// const PART_SIZE = 5 * 1000 * 1000 * 1000;
18+
const PART_SIZE = 5 * 1024 * 1024 * 1024;
19+
20+
/**
21+
* @var \Pterodactyl\Repositories\Eloquent\BackupRepository
22+
*/
23+
private $repository;
24+
25+
/**
26+
* @var \Pterodactyl\Extensions\Backups\BackupManager
27+
*/
28+
private $backupManager;
29+
30+
/**
31+
* BackupRemoteUploadController constructor.
32+
*
33+
* @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository
34+
* @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager
35+
*/
36+
public function __construct(BackupRepository $repository, BackupManager $backupManager)
37+
{
38+
$this->repository = $repository;
39+
$this->backupManager = $backupManager;
40+
}
41+
42+
/**
43+
* ?
44+
*
45+
* @param \Illuminate\Http\Request $request
46+
* @param string $backup
47+
*
48+
* @return \Illuminate\Http\JsonResponse
49+
*
50+
* @throws \Pterodactyl\Exceptions\Repository\RecordNotFoundException
51+
* @throws \Exception
52+
*/
53+
public function __invoke(Request $request, string $backup)
54+
{
55+
$size = $request->query('size', null);
56+
if ($size === null) {
57+
return new JsonResponse([], JsonResponse::HTTP_BAD_REQUEST);
58+
}
59+
60+
/** @var \Pterodactyl\Models\Backup $model */
61+
$model = $this->repository->findFirstWhere([[ 'uuid', '=', $backup ]]);
62+
63+
// Prevent backups that have already been completed from trying to
64+
// be uploaded again.
65+
if (! is_null($model->completed_at)) {
66+
return new JsonResponse([], JsonResponse::HTTP_CONFLICT);
67+
}
68+
69+
// Ensure we are using the S3 adapter.
70+
$adapter = $this->backupManager->adapter();
71+
if (! $adapter instanceof AwsS3Adapter) {
72+
return new JsonResponse([], JsonResponse::HTTP);
73+
}
74+
75+
$path = sprintf('%s/%s.tar.gz', $model->server->uuid, $model->uuid);
76+
77+
$client = $adapter->getClient();
78+
79+
$result = $client->execute($client->getCommand('CreateMultipartUpload', [
80+
'Bucket' => $adapter->getBucket(),
81+
'Key' => $path,
82+
'ContentType' => 'application/x-gzip',
83+
]));
84+
$uploadId = $result->get('UploadId');
85+
86+
$completeMultipartUpload = $client->createPresignedRequest(
87+
$client->getCommand('CompleteMultipartUpload', [
88+
'Bucket' => $adapter->getBucket(),
89+
'Key' => $path,
90+
'ContentType' => 'application/x-gzip',
91+
'UploadId' => $uploadId,
92+
]),
93+
CarbonImmutable::now()->addMinutes(30)
94+
);
95+
96+
$abortMultipartUpload = $client->createPresignedRequest(
97+
$client->getCommand('AbortMultipartUpload', [
98+
'Bucket' => $adapter->getBucket(),
99+
'Key' => $path,
100+
'ContentType' => 'application/x-gzip',
101+
'UploadId' => $uploadId,
102+
]),
103+
CarbonImmutable::now()->addMinutes(45)
104+
);
105+
106+
$partCount = (int) $size / (self::PART_SIZE);
107+
108+
$parts = [];
109+
for ($i = 0; $i < $partCount; $i++) {
110+
$part = $client->createPresignedRequest(
111+
$client->getCommand('UploadPart', [
112+
'Bucket' => $adapter->getBucket(),
113+
'Key' => $path,
114+
'ContentType' => 'application/x-gzip',
115+
'UploadId' => $uploadId,
116+
'PartNumber' => $i + 1,
117+
]),
118+
CarbonImmutable::now()->addMinutes(30)
119+
);
120+
121+
array_push($parts, $part->getUri()->__toString());
122+
}
123+
124+
return new JsonResponse([
125+
'CompleteMultipartUpload' => $completeMultipartUpload->getUri()->__toString(),
126+
'AbortMultipartUpload' => $abortMultipartUpload->getUri()->__toString(),
127+
'Parts' => $parts,
128+
'PartSize' => self::PART_SIZE,
129+
], JsonResponse::HTTP_OK);
130+
}
131+
}

‎app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@
22

33
namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
44

5-
use Carbon\Carbon;
65
use Carbon\CarbonImmutable;
76
use Illuminate\Http\JsonResponse;
87
use Pterodactyl\Http\Controllers\Controller;
98
use Pterodactyl\Repositories\Eloquent\BackupRepository;
10-
use Pterodactyl\Exceptions\Http\HttpForbiddenException;
119
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
1210
use Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest;
1311

@@ -42,7 +40,7 @@ public function __invoke(ReportBackupCompleteRequest $request, string $backup)
4240
/** @var \Pterodactyl\Models\Backup $model */
4341
$model = $this->repository->findFirstWhere([[ 'uuid', '=', $backup ]]);
4442

45-
if (!is_null($model->completed_at)) {
43+
if (! is_null($model->completed_at)) {
4644
throw new BadRequestHttpException(
4745
'Cannot update the status of a backup that is already marked as completed.'
4846
);

‎app/Models/Backup.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
/**
88
* @property int $id
99
* @property int $server_id
10-
* @property int $uuid
10+
* @property string $uuid
1111
* @property bool $is_successful
1212
* @property string $name
1313
* @property string[] $ignored_files

‎app/Repositories/Wings/DaemonBackupRepository.php‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,11 @@ public function setBackupAdapter(string $adapter)
3333
* Tells the remote Daemon to begin generating a backup for the server.
3434
*
3535
* @param \Pterodactyl\Models\Backup $backup
36-
* @param string|null $presignedUrl
3736
* @return \Psr\Http\Message\ResponseInterface
3837
*
3938
* @throws \Pterodactyl\Exceptions\Http\Connection\DaemonConnectionException
4039
*/
41-
public function backup(Backup $backup, string $presignedUrl = null): ResponseInterface
40+
public function backup(Backup $backup): ResponseInterface
4241
{
4342
Assert::isInstanceOf($this->server, Server::class);
4443

@@ -50,7 +49,6 @@ public function backup(Backup $backup, string $presignedUrl = null): ResponseInt
5049
'adapter' => $this->adapter ?? config('backups.default'),
5150
'uuid' => $backup->uuid,
5251
'ignored_files' => $backup->ignored_files,
53-
'presigned_url' => $presignedUrl,
5452
],
5553
]
5654
);

‎app/Services/Backups/InitiateBackupService.php‎

Lines changed: 1 addition & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
use Webmozart\Assert\Assert;
88
use Pterodactyl\Models\Backup;
99
use Pterodactyl\Models\Server;
10-
use League\Flysystem\AwsS3v3\AwsS3Adapter;
1110
use Illuminate\Database\ConnectionInterface;
1211
use Pterodactyl\Extensions\Backups\BackupManager;
1312
use Pterodactyl\Repositories\Eloquent\BackupRepository;
@@ -122,42 +121,11 @@ public function handle(Server $server, string $name = null): Backup
122121
'disk' => $this->backupManager->getDefaultAdapter(),
123122
], true, true);
124123

125-
$url = $this->getS3PresignedUrl(sprintf('%s/%s.tar.gz', $server->uuid, $backup->uuid));
126-
127124
$this->daemonBackupRepository->setServer($server)
128125
->setBackupAdapter($this->backupManager->getDefaultAdapter())
129-
->backup($backup, $url);
126+
->backup($backup);
130127

131128
return $backup;
132129
});
133130
}
134-
135-
/**
136-
* Generates a presigned URL for the wings daemon to upload the completed archive
137-
* to. We use a 30 minute expiration on these URLs to avoid issues with large backups
138-
* that may take some time to complete.
139-
*
140-
* @param string $path
141-
* @return string|null
142-
*/
143-
protected function getS3PresignedUrl(string $path)
144-
{
145-
$adapter = $this->backupManager->adapter();
146-
if (! $adapter instanceof AwsS3Adapter) {
147-
return null;
148-
}
149-
150-
$client = $adapter->getClient();
151-
152-
$request = $client->createPresignedRequest(
153-
$client->getCommand('PutObject', [
154-
'Bucket' => $adapter->getBucket(),
155-
'Key' => $path,
156-
'ContentType' => 'application/x-gzip',
157-
]),
158-
CarbonImmutable::now()->addMinutes(30)
159-
);
160-
161-
return $request->getUri()->__toString();
162-
}
163131
}

‎resources/scripts/components/server/backups/BackupRow.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export default ({ backup, className }: Props) => {
2929
items: data.items.map(b => b.uuid !== backup.uuid ? b : ({
3030
...b,
3131
isSuccessful: parsed.is_successful || true,
32-
checksum: parsed.checksum || '',
32+
checksum: (parsed.checksum_type || '') + ':' + (parsed.checksum || ''),
3333
bytes: parsed.file_size || 0,
3434
completedAt: new Date(),
3535
})),

‎routes/api-remote.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,6 @@
1818
});
1919

2020
Route::group(['prefix' => '/backups'], function () {
21+
Route::get('/{backup}', 'Backups\BackupRemoteUploadController');
2122
Route::post('/{backup}', 'Backups\BackupStatusController');
2223
});

0 commit comments

Comments
 (0)