s3 backups: handle CompleteMultipartUpload and AbortMultipartUpload on the panel instead of in wings, add BACKUP_PRESIGNED_URL_LIFESPAN environment variable

Author: matthewpi

app/Extensions/Backups/BackupManager.php

@@ -9,6 +9,7 @@
 use Webmozart\Assert\Assert;
 use InvalidArgumentException;
 use League\Flysystem\AdapterInterface;
+use Illuminate\Foundation\Application;
 use League\Flysystem\AwsS3v3\AwsS3Adapter;
 use League\Flysystem\Memory\MemoryAdapter;
 use Illuminate\Contracts\Config\Repository;
@@ -44,7 +45,7 @@ class BackupManager
      *
      * @param \Illuminate\Foundation\Application $app
      */
-    public function __construct($app)
+    public function __construct(Application $app)
     {
         $this->app = $app;
         $this->config = $app->make(Repository::class);

app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php

@@ -6,7 +6,9 @@
 use Illuminate\Http\Request;
 use Pterodactyl\Models\Backup;
 use Illuminate\Http\JsonResponse;
+use Illuminate\Foundation\Application;
 use League\Flysystem\AwsS3v3\AwsS3Adapter;
+use Illuminate\Contracts\Config\Repository;
 use Pterodactyl\Http\Controllers\Controller;
 use Pterodactyl\Extensions\Backups\BackupManager;
 use Pterodactyl\Repositories\Eloquent\BackupRepository;
@@ -16,6 +18,11 @@ class BackupRemoteUploadController extends Controller
 {
     const PART_SIZE = 5 * 1024 * 1024 * 1024;
 
+    /**
+     * @var \Illuminate\Contracts\Config\Repository
+     */
+    protected $config;
+
     /**
      * @var \Pterodactyl\Repositories\Eloquent\BackupRepository
      */
@@ -29,11 +36,13 @@ class BackupRemoteUploadController extends Controller
     /**
      * BackupRemoteUploadController constructor.
      *
+     * @param \Illuminate\Foundation\Application $app
      * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository
      * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager
      */
-    public function __construct(BackupRepository $repository, BackupManager $backupManager)
+    public function __construct(Application $app, BackupRepository $repository, BackupManager $backupManager)
     {
+        $this->config = $app->make(Repository::class);
         $this->repository = $repository;
         $this->backupManager = $backupManager;
     }
@@ -69,15 +78,15 @@ public function __invoke(Request $request, string $backup)
         // Ensure we are using the S3 adapter.
         $adapter = $this->backupManager->adapter();
         if (! $adapter instanceof AwsS3Adapter) {
-            throw new BadRequestHttpException('The configured backup adapter is not an S3 compatiable adapter.');
+            throw new BadRequestHttpException('The configured backup adapter is not an S3 compatible adapter.');
         }
 
         // The path where backup will be uploaded to
         $path = sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid);
 
         // Get the S3 client
         $client = $adapter->getClient();
-        $expires = CarbonImmutable::now()->addMinutes(30);
+        $expires = CarbonImmutable::now()->addMinutes($this->config->get('backups.presigned_url_lifespan', 60));
 
         // Params for generating the presigned urls
         $params = [
@@ -102,14 +111,9 @@ public function __invoke(Request $request, string $backup)
         }
 
         return new JsonResponse([
+            'upload_id' => $params['UploadId'],
             'parts' => $parts,
             'part_size' => self::PART_SIZE,
-            'complete_multipart_upload' => $client->createPresignedRequest(
-                $client->getCommand('CompleteMultipartUpload', $params), $expires
-            )->getUri()->__toString(),
-            'abort_multipart_upload' => $client->createPresignedRequest(
-                $client->getCommand('AbortMultipartUpload', $params), $expires->addMinutes(15)
-            )->getUri()->__toString(),
         ]);
     }
 }

app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php

@@ -3,8 +3,11 @@
 namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
 
 use Carbon\CarbonImmutable;
+use Pterodactyl\Models\Backup;
 use Illuminate\Http\JsonResponse;
+use League\Flysystem\AwsS3v3\AwsS3Adapter;
 use Pterodactyl\Http\Controllers\Controller;
+use Pterodactyl\Extensions\Backups\BackupManager;
 use Pterodactyl\Repositories\Eloquent\BackupRepository;
 use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
 use Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest;
@@ -16,24 +19,33 @@ class BackupStatusController extends Controller
      */
     private $repository;
 
+    /**
+     * @var \Pterodactyl\Extensions\Backups\BackupManager
+     */
+    private $backupManager;
+
     /**
      * BackupStatusController constructor.
      *
      * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository
+     * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager
      */
-    public function __construct(BackupRepository $repository)
+    public function __construct(BackupRepository $repository, BackupManager $backupManager)
     {
         $this->repository = $repository;
+        $this->backupManager = $backupManager;
     }
 
     /**
      * Handles updating the state of a backup.
      *
      * @param \Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest $request
      * @param string $backup
+     *
      * @return \Illuminate\Http\JsonResponse
      *
      * @throws \Pterodactyl\Exceptions\Repository\RecordNotFoundException
+     * @throws \Exception
      */
     public function __invoke(ReportBackupCompleteRequest $request, string $backup)
     {
@@ -47,13 +59,42 @@ public function __invoke(ReportBackupCompleteRequest $request, string $backup)
         }
 
         $successful = $request->input('successful') ? true : false;
+
+        // TODO: Still run s3 code even if this fails.
         $model->forceFill([
             'is_successful' => $successful,
             'checksum' => $successful ? ($request->input('checksum_type') . ':' . $request->input('checksum')) : null,
             'bytes' => $successful ? $request->input('size') : 0,
             'completed_at' => CarbonImmutable::now(),
         ])->save();
 
+        // Check if we are using the s3 backup adapter.
+        $adapter = $this->backupManager->adapter();
+        if ($adapter instanceof AwsS3Adapter) {
+            /** @var \Pterodactyl\Models\Backup $backup */
+            $backup = Backup::query()->where('uuid', $backup)->firstOrFail();
+
+            $client = $adapter->getClient();
+
+            $params = [
+                'Bucket' => $adapter->getBucket(),
+                'Key' => sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid),
+                'UploadId' => $request->input('upload_id'),
+            ];
+
+            // If the backup was not successful, send an AbortMultipartUpload request.
+            if (! $successful) {
+                $client->execute($client->getCommand('AbortMultipartUpload', $params));
+                return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT);
+            }
+
+            // Otherwise send a CompleteMultipartUpload request.
+            $params['MultipartUpload'] = [
+                'Parts' => $client->execute($client->getCommand('ListParts', $params))['Parts'],
+            ];
+            $client->execute($client->getCommand('CompleteMultipartUpload', $params));
+        }
+
         return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT);
     }
 }

app/Services/Backups/InitiateBackupService.php

@@ -99,6 +99,8 @@ public function setIgnoredFiles(?array $ignored)
      *
      * @param \Pterodactyl\Models\Server $server
      * @param string|null $name
+     * @param bool $override
+     *
      * @return \Pterodactyl\Models\Backup
      *
      * @throws \Throwable
@@ -107,13 +109,13 @@ public function setIgnoredFiles(?array $ignored)
      */
     public function handle(Server $server, string $name = null, bool $override = false): Backup
     {
-        $previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10);
+        /*$previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10);
         if ($previous->count() >= 2) {
             throw new TooManyRequestsHttpException(
                 CarbonImmutable::now()->diffInSeconds($previous->last()->created_at->addMinutes(10)),
                 'Only two backups may be generated within a 10 minute span of time.'
             );
-        }
+        }*/
 
         // Check if the server has reached or exceeded it's backup limit
         if (!$server->backup_limit || $server->backups()->where('is_successful', true)->count() >= $server->backup_limit) {

config/backups.php

@@ -8,6 +8,10 @@
     // have been made, without losing data.
     'default' => env('APP_BACKUP_DRIVER', Backup::ADAPTER_WINGS),
 
+    // This value is used to determine the lifespan of UploadPart presigned urls that wings
+    // uses to upload backups to S3 storage.  Value is in minutes, so this would default to an hour.
+    'presigned_url_lifespan' => env('BACKUP_PRESIGNED_URL_LIFESPAN', 60),
+
     'disks' => [
         // There is no configuration for the local disk for Wings. That configuration
         // is determined by the Daemon configuration, and not the Panel.