Skip to content

Commit 6eff9d6

Browse files
DaneEverittDaneEveritt
authored
Merge pull request pterodactyl#2634 from pterodactyl/issue/2599
Switch to s3 multipart uploads for backups
2 parents 23d2352 + 6af848c commit 6eff9d6

7 files changed

Lines changed: 150 additions & 41 deletions

File tree

‎app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
<?php
2+
3+
namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
4+
5+
use Carbon\CarbonImmutable;
6+
use Illuminate\Http\Request;
7+
use Pterodactyl\Models\Backup;
8+
use Illuminate\Http\JsonResponse;
9+
use League\Flysystem\AwsS3v3\AwsS3Adapter;
10+
use Pterodactyl\Http\Controllers\Controller;
11+
use Pterodactyl\Extensions\Backups\BackupManager;
12+
use Pterodactyl\Repositories\Eloquent\BackupRepository;
13+
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
14+
15+
class BackupRemoteUploadController extends Controller
16+
{
17+
const PART_SIZE = 5 * 1024 * 1024 * 1024;
18+
19+
/**
20+
* @var \Pterodactyl\Repositories\Eloquent\BackupRepository
21+
*/
22+
private $repository;
23+
24+
/**
25+
* @var \Pterodactyl\Extensions\Backups\BackupManager
26+
*/
27+
private $backupManager;
28+
29+
/**
30+
* BackupRemoteUploadController constructor.
31+
*
32+
* @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository
33+
* @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager
34+
*/
35+
public function __construct(BackupRepository $repository, BackupManager $backupManager)
36+
{
37+
$this->repository = $repository;
38+
$this->backupManager = $backupManager;
39+
}
40+
41+
/**
42+
* Returns the required presigned urls to upload a backup to S3 cloud storage.
43+
*
44+
* @param \Illuminate\Http\Request $request
45+
* @param string $backup
46+
*
47+
* @return \Illuminate\Http\JsonResponse
48+
*
49+
* @throws \Exception
50+
* @throws \Illuminate\Database\Eloquent\ModelNotFoundException
51+
*/
52+
public function __invoke(Request $request, string $backup)
53+
{
54+
// Get the size query parameter.
55+
$size = $request->query('size', null);
56+
if (is_null($size)) {
57+
throw new BadRequestHttpException('Missing size query parameter.');
58+
}
59+
60+
/** @var \Pterodactyl\Models\Backup $model */
61+
$model = Backup::query()->where([[ 'uuid', '=', $backup ]])->firstOrFail();
62+
63+
// Prevent backups that have already been completed from trying to
64+
// be uploaded again.
65+
if (! is_null($model->completed_at)) {
66+
return new JsonResponse([], JsonResponse::HTTP_CONFLICT);
67+
}
68+
69+
// Ensure we are using the S3 adapter.
70+
$adapter = $this->backupManager->adapter();
71+
if (! $adapter instanceof AwsS3Adapter) {
72+
throw new BadRequestHttpException('Backups are not using the s3 storage driver');
73+
}
74+
75+
// The path where backup will be uploaded to
76+
$path = sprintf('%s/%s.tar.gz', $model->server->uuid, $model->uuid);
77+
78+
// Get the S3 client
79+
$client = $adapter->getClient();
80+
81+
// Params for generating the presigned urls
82+
$params = [
83+
'Bucket' => $adapter->getBucket(),
84+
'Key' => $path,
85+
'ContentType' => 'application/x-gzip',
86+
];
87+
88+
// Execute the CreateMultipartUpload request
89+
$result = $client->execute($client->getCommand('CreateMultipartUpload', $params));
90+
91+
// Get the UploadId from the CreateMultipartUpload request,
92+
// this is needed to create the other presigned urls
93+
$uploadId = $result->get('UploadId');
94+
95+
// Create a CompleteMultipartUpload presigned url
96+
$completeMultipartUpload = $client->createPresignedRequest(
97+
$client->getCommand(
98+
'CompleteMultipartUpload',
99+
array_merge($params, [
100+
'UploadId' => $uploadId,
101+
])
102+
),
103+
CarbonImmutable::now()->addMinutes(30)
104+
);
105+
106+
// Create a AbortMultipartUpload presigned url
107+
$abortMultipartUpload = $client->createPresignedRequest(
108+
$client->getCommand(
109+
'AbortMultipartUpload',
110+
array_merge($params, [
111+
'UploadId' => $uploadId,
112+
])
113+
),
114+
CarbonImmutable::now()->addMinutes(45)
115+
);
116+
117+
// Calculate the number of parts needed to upload the backup
118+
$partCount = (int) $size / (self::PART_SIZE);
119+
120+
// Create as many UploadPart presigned urls as needed
121+
$parts = [];
122+
for ($i = 0; $i < $partCount; $i++) {
123+
$part = $client->createPresignedRequest(
124+
$client->getCommand(
125+
'UploadPart',
126+
array_merge($params, [
127+
'UploadId' => $uploadId,
128+
'PartNumber' => $i + 1,
129+
])
130+
),
131+
CarbonImmutable::now()->addMinutes(30)
132+
);
133+
134+
array_push($parts, $part->getUri()->__toString());
135+
}
136+
137+
return new JsonResponse([
138+
'complete_multipart_upload' => $completeMultipartUpload->getUri()->__toString(),
139+
'abort_multipart_upload' => $abortMultipartUpload->getUri()->__toString(),
140+
'parts' => $parts,
141+
'part_size' => self::PART_SIZE,
142+
]);
143+
}
144+
}

‎app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@
22

33
namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
44

5-
use Carbon\Carbon;
65
use Carbon\CarbonImmutable;
76
use Illuminate\Http\JsonResponse;
87
use Pterodactyl\Http\Controllers\Controller;
98
use Pterodactyl\Repositories\Eloquent\BackupRepository;
10-
use Pterodactyl\Exceptions\Http\HttpForbiddenException;
119
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
1210
use Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest;
1311

@@ -42,7 +40,7 @@ public function __invoke(ReportBackupCompleteRequest $request, string $backup)
4240
/** @var \Pterodactyl\Models\Backup $model */
4341
$model = $this->repository->findFirstWhere([[ 'uuid', '=', $backup ]]);
4442

45-
if (!is_null($model->completed_at)) {
43+
if (! is_null($model->completed_at)) {
4644
throw new BadRequestHttpException(
4745
'Cannot update the status of a backup that is already marked as completed.'
4846
);

‎app/Models/Backup.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
/**
88
* @property int $id
99
* @property int $server_id
10-
* @property int $uuid
10+
* @property string $uuid
1111
* @property bool $is_successful
1212
* @property string $name
1313
* @property string[] $ignored_files

‎app/Repositories/Wings/DaemonBackupRepository.php‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,11 @@ public function setBackupAdapter(string $adapter)
3333
* Tells the remote Daemon to begin generating a backup for the server.
3434
*
3535
* @param \Pterodactyl\Models\Backup $backup
36-
* @param string|null $presignedUrl
3736
* @return \Psr\Http\Message\ResponseInterface
3837
*
3938
* @throws \Pterodactyl\Exceptions\Http\Connection\DaemonConnectionException
4039
*/
41-
public function backup(Backup $backup, string $presignedUrl = null): ResponseInterface
40+
public function backup(Backup $backup): ResponseInterface
4241
{
4342
Assert::isInstanceOf($this->server, Server::class);
4443

@@ -50,7 +49,6 @@ public function backup(Backup $backup, string $presignedUrl = null): ResponseInt
5049
'adapter' => $this->adapter ?? config('backups.default'),
5150
'uuid' => $backup->uuid,
5251
'ignored_files' => $backup->ignored_files,
53-
'presigned_url' => $presignedUrl,
5452
],
5553
]
5654
);

‎app/Services/Backups/InitiateBackupService.php‎

Lines changed: 1 addition & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
use Webmozart\Assert\Assert;
88
use Pterodactyl\Models\Backup;
99
use Pterodactyl\Models\Server;
10-
use League\Flysystem\AwsS3v3\AwsS3Adapter;
1110
use Illuminate\Database\ConnectionInterface;
1211
use Pterodactyl\Extensions\Backups\BackupManager;
1312
use Pterodactyl\Repositories\Eloquent\BackupRepository;
@@ -122,42 +121,11 @@ public function handle(Server $server, string $name = null): Backup
122121
'disk' => $this->backupManager->getDefaultAdapter(),
123122
], true, true);
124123

125-
$url = $this->getS3PresignedUrl(sprintf('%s/%s.tar.gz', $server->uuid, $backup->uuid));
126-
127124
$this->daemonBackupRepository->setServer($server)
128125
->setBackupAdapter($this->backupManager->getDefaultAdapter())
129-
->backup($backup, $url);
126+
->backup($backup);
130127

131128
return $backup;
132129
});
133130
}
134-
135-
/**
136-
* Generates a presigned URL for the wings daemon to upload the completed archive
137-
* to. We use a 30 minute expiration on these URLs to avoid issues with large backups
138-
* that may take some time to complete.
139-
*
140-
* @param string $path
141-
* @return string|null
142-
*/
143-
protected function getS3PresignedUrl(string $path)
144-
{
145-
$adapter = $this->backupManager->adapter();
146-
if (! $adapter instanceof AwsS3Adapter) {
147-
return null;
148-
}
149-
150-
$client = $adapter->getClient();
151-
152-
$request = $client->createPresignedRequest(
153-
$client->getCommand('PutObject', [
154-
'Bucket' => $adapter->getBucket(),
155-
'Key' => $path,
156-
'ContentType' => 'application/x-gzip',
157-
]),
158-
CarbonImmutable::now()->addMinutes(30)
159-
);
160-
161-
return $request->getUri()->__toString();
162-
}
163131
}

‎resources/scripts/components/server/backups/BackupRow.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export default ({ backup, className }: Props) => {
2929
items: data.items.map(b => b.uuid !== backup.uuid ? b : ({
3030
...b,
3131
isSuccessful: parsed.is_successful || true,
32-
checksum: parsed.checksum || '',
32+
checksum: (parsed.checksum_type || '') + ':' + (parsed.checksum || ''),
3333
bytes: parsed.file_size || 0,
3434
completedAt: new Date(),
3535
})),

‎routes/api-remote.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,6 @@
1818
});
1919

2020
Route::group(['prefix' => '/backups'], function () {
21+
Route::get('/{backup}', 'Backups\BackupRemoteUploadController');
2122
Route::post('/{backup}', 'Backups\BackupStatusController');
2223
});

0 commit comments

Comments
 (0)