Add endpoint logic necessary to reset server states if they get stuck installing/restoring when wings restarts

Author: DaneEveritt

app/Http/Controllers/Api/Remote/Servers/ServerDetailsController.php

@@ -4,7 +4,10 @@
 
 use Illuminate\Http\Request;
 use Pterodactyl\Models\Server;
+use Pterodactyl\Models\AuditLog;
 use Illuminate\Http\JsonResponse;
+use Illuminate\Database\Query\Builder;
+use Illuminate\Database\Query\JoinClause;
 use Pterodactyl\Http\Controllers\Controller;
 use Pterodactyl\Repositories\Eloquent\NodeRepository;
 use Pterodactyl\Services\Eggs\EggConfigurationService;
@@ -83,4 +86,67 @@ public function list(Request $request)
 
         return new ServerConfigurationCollection($servers);
     }
+
+    /**
+     * Resets the state of all servers on the node to be normal. This is triggered
+     * when Wings restarts and is useful for ensuring that any servers on the node
+     * do not get incorrectly stuck in installing/restoring from backup states since
+     * a Wings reboot would completely stop those processes.
+     *
+     * @param \Illuminate\Http\Request $request
+     * @return \Illuminate\Http\JsonResponse
+     *
+     * @throws \Throwable
+     */
+    public function resetState(Request $request)
+    {
+        $node = $request->attributes->get('node');
+
+        // Get all of the servers that are currently marked as restoring from a backup
+        // on this node that do not have a failed backup tracked in the audit logs table
+        // as well.
+        //
+        // For each of those servers we'll track a new audit log entry to mark them as
+        // failed and then update them all to be in a valid state.
+        /** @var \Pterodactyl\Models\Server[] $servers */
+        $servers = Server::query()
+            ->select('servers.*')
+            ->selectRaw('started.metadata->>"$.backup_uuid" as backup_uuid')
+            ->leftJoinSub(function (Builder $builder) {
+                $builder->select('*')->from('audit_logs')
+                    ->where('action', AuditLog::SERVER__BACKUP_RESTORE_STARTED)
+                    ->orderByDesc('created_at')
+                    ->limit(1);
+            }, 'started', 'started.server_id', '=', 'servers.id')
+            ->leftJoin('audit_logs as completed', function (JoinClause $clause) {
+                $clause->whereColumn('completed.created_at', '>', 'started.created_at')
+                    ->whereIn('completed.action', [
+                        AuditLog::SERVER__BACKUP_RESTORE_COMPLETED,
+                        AuditLog::SERVER__BACKUP_RESTORE_FAILED,
+                    ]);
+            })
+            ->whereNotNull('started.id')
+            ->whereNull('completed.id')
+            ->where('servers.node_id', $node->id)
+            ->where('servers.status', Server::STATUS_RESTORING_BACKUP)
+            ->get();
+
+        foreach ($servers as $server) {
+            // Just create a new audit entry for this event and update the server state
+            // so that power actions, file management, and backups can resume as normal.
+            $server->audit(AuditLog::SERVER__BACKUP_RESTORE_FAILED, function (AuditLog $audit, Server $server) {
+                $audit->is_system = true;
+                $audit->metadata = ['backup_uuid' => $server->getAttribute('backup_uuid')];
+                $server->update(['status' => null]);
+            });
+        }
+
+        // Update any server marked as installing or restoring as being in a normal state
+        // at this point in the process.
+        Server::query()->where('node_id', $node->id)
+            ->whereIn('status', [Server::STATUS_INSTALLING, Server::STATUS_RESTORING_BACKUP])
+            ->update(['status' => null]);
+
+        return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT);
+    }
 }

database/migrations/2021_02_23_205021_add_index_for_server_and_action.php

@@ -0,0 +1,39 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddIndexForServerAndAction extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('audit_logs', function (Blueprint $table) {
+            // Doing the index in this order lets me use the action alone without the server
+            // or I can later include the server to also filter down at an even more specific
+            // level.
+            //
+            // Ordering the other way around would require a second index for only "action" in
+            // order to query a specific action type for any server. Remeber, indexes run left
+            // to right in MySQL.
+            $table->index(['action', 'server_id']);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('audit_logs', function (Blueprint $table) {
+            $table->dropIndex(['action', 'server_id']);
+        });
+    }
+}

routes/api-remote.php

@@ -6,6 +6,7 @@
 Route::post('/sftp/auth', 'SftpAuthenticationController');
 
 Route::get('/servers', 'Servers\ServerDetailsController@list');
+Route::post('/servers/reset', 'Servers\ServerDetailsController@resetState');
 
 Route::group(['prefix' => '/servers/{uuid}'], function () {
     Route::get('/', 'Servers\ServerDetailsController');