Add database keep-alive for crashed standalone DBs

backend/app/Console/Kernel.php

@@ -13,6 +13,7 @@
 use App\Jobs\DevForge\ApplicationKeepAliveJob;
 use App\Jobs\DevForge\ApplicationReadinessWatchdogJob;
 use App\Jobs\DevForge\CheckDockerImageUpdatesJob;
+use App\Jobs\DevForge\DatabaseKeepAliveJob;
 use App\Jobs\DevForge\InstanceHostDiskGuardJob;
 use App\Jobs\PullChangelog;
 use App\Jobs\PullTemplatesFromCDN;
@@ -85,6 +86,10 @@ protected function schedule(Schedule $schedule): void
                 ->everyTwoMinutes()
                 ->onOneServer();
 
+            $this->scheduleInstance->job(new DatabaseKeepAliveJob)
+                ->everyTwoMinutes()
+                ->onOneServer();
+
             $this->scheduleInstance->job(new InstanceHostDiskGuardJob)
                 ->everyFiveMinutes()
                 ->onOneServer();
@@ -132,6 +137,10 @@ protected function schedule(Schedule $schedule): void
                 ->everyTwoMinutes()
                 ->onOneServer();
 
+            $this->scheduleInstance->job(new DatabaseKeepAliveJob)
+                ->everyTwoMinutes()
+                ->onOneServer();
+
             $this->scheduleInstance->job(new InstanceHostDiskGuardJob)
                 ->everyFiveMinutes()
                 ->onOneServer();

backend/app/Jobs/DevForge/DatabaseKeepAliveJob.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace App\Jobs\DevForge;
+
+use App\Services\DevForge\Database\DatabaseKeepAliveService;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class DatabaseKeepAliveJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 1;
+
+    public int $timeout = 180;
+
+    public function __construct()
+    {
+        $this->onQueue('default');
+    }
+
+    public function handle(DatabaseKeepAliveService $keepAlive): void
+    {
+        $keepAlive->tickAllTeams();
+    }
+}

backend/app/Services/DevForge/Core/CoreResourceAction.php

@@ -12,6 +12,7 @@
 use App\Models\Application;
 use App\Models\Service;
 use App\Services\DevForge\Application\ApplicationDesiredRuntimeState;
+use App\Services\DevForge\Database\DatabaseDesiredRuntimeState;
 use Illuminate\Database\Eloquent\Model;
 use RuntimeException;
 use Symfony\Component\HttpKernel\Exception\HttpException;
@@ -21,6 +22,7 @@ class CoreResourceAction
 {
     public function __construct(
         private readonly ApplicationDesiredRuntimeState $desiredRuntimeState,
+        private readonly DatabaseDesiredRuntimeState $databaseDesiredRuntimeState,
     ) {}
 
     /**
@@ -117,6 +119,12 @@ private function database(Model $database, string $action, array $options): arra
             throw new HttpException(422, $exception->getMessage(), previous: $exception);
         }
 
+        if ($action === 'stop') {
+            $this->databaseDesiredRuntimeState->markDesiredStopped($database);
+        } else {
+            $this->databaseDesiredRuntimeState->markDesiredRunning($database);
+        }
+
         return [
             'queued' => false,
             'completed' => true,

backend/app/Services/DevForge/Database/DatabaseDesiredRuntimeState.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace App\Services\DevForge\Database;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Facades\Cache;
+
+class DatabaseDesiredRuntimeState
+{
+    private const CACHE_PREFIX = 'devforge:database_keep_alive:desired:';
+
+    public function markDesiredRunning(Model $database): void
+    {
+        Cache::put(
+            $this->cacheKey((string) $database->uuid),
+            true,
+            now()->addSeconds($this->ttlSeconds()),
+        );
+    }
+
+    public function markDesiredStopped(Model $database): void
+    {
+        Cache::put(
+            $this->cacheKey((string) $database->uuid),
+            false,
+            now()->addSeconds($this->ttlSeconds()),
+        );
+    }
+
+    /**
+     * Explicit cache value: true = must run, false = stopped on purpose, null = unknown.
+     */
+    public function cachedDesired(Model $database): ?bool
+    {
+        $cached = Cache::get($this->cacheKey((string) $database->uuid));
+
+        if ($cached === true) {
+            return true;
+        }
+
+        if ($cached === false) {
+            return false;
+        }
+
+        return null;
+    }
+
+    public function isDesiredRunning(Model $database): bool
+    {
+        $cached = $this->cachedDesired($database);
+
+        if ($cached === true) {
+            return true;
+        }
+
+        if ($cached === false) {
+            return false;
+        }
+
+        return $this->isRunningStatus((string) ($database->status ?? ''));
+    }
+
+    private function cacheKey(string $uuid): string
+    {
+        return self::CACHE_PREFIX.$uuid;
+    }
+
+    private function ttlSeconds(): int
+    {
+        return max(3600, (int) config('devforge.database_keep_alive.desired_ttl_seconds', 60 * 60 * 24 * 30));
+    }
+
+    private function isRunningStatus(string $status): bool
+    {
+        return str($status)->before(':')->lower()->trim()->value() === 'running';
+    }
+}

backend/app/Services/DevForge/Database/DatabaseKeepAliveService.php

@@ -0,0 +1,160 @@
+<?php
+
+namespace App\Services\DevForge\Database;
+
+use App\Models\Application;
+use App\Models\EnvironmentVariable;
+use App\Models\Team;
+use App\Services\DevForge\Core\CoreResourceCatalog;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Facades\Log;
+
+class DatabaseKeepAliveService
+{
+    public function __construct(
+        private readonly CoreResourceCatalog $catalog,
+        private readonly StandaloneDatabaseRuntimeGuard $runtimeGuard,
+        private readonly DatabaseDesiredRuntimeState $desiredRuntimeState,
+    ) {}
+
+    public function enabled(): bool
+    {
+        return (bool) config('devforge.database_keep_alive.enabled', true)
+            && (bool) config('devforge.enabled', true);
+    }
+
+    public function tickAllTeams(): void
+    {
+        if (! $this->enabled()) {
+            return;
+        }
+
+        Team::query()
+            ->select(['id'])
+            ->orderBy('id')
+            ->each(function (Team $team): void {
+                try {
+                    $this->tickTeam($team);
+                } catch (\Throwable $exception) {
+                    Log::warning('DevForge database keep-alive tick failed.', [
+                        'team_id' => $team->id,
+                        'error' => $exception->getMessage(),
+                    ]);
+                }
+            });
+    }
+
+    /**
+     * @return array{restarted: array<int, string>}
+     */
+    public function tickTeam(Team $team): array
+    {
+        if (! $this->enabled()) {
+            return ['restarted' => []];
+        }
+
+        $databases = $this->catalog->resources($team, 'databases');
+        if ($databases->isEmpty()) {
+            return ['restarted' => []];
+        }
+
+        $restarted = [];
+
+        foreach ($databases as $database) {
+            if (! $database instanceof Model || blank($database->uuid)) {
+                continue;
+            }
+
+            $status = (string) ($database->status ?? 'unknown');
+
+            if ($this->isRunningStatus($status) || $this->isStartingStatus($status)) {
+                $this->desiredRuntimeState->markDesiredRunning($database);
+
+                continue;
+            }
+
+            if (! $this->isStoppedStatus($status)) {
+                continue;
+            }
+
+            if (! $this->shouldRestart($database)) {
+                continue;
+            }
+
+            try {
+                if ($this->runtimeGuard->ensureRunning($database)) {
+                    $this->desiredRuntimeState->markDesiredRunning($database);
+                    $restarted[] = (string) $database->uuid;
+                }
+            } catch (\Throwable $exception) {
+                Log::warning('DevForge database keep-alive failed to start database.', [
+                    'team_id' => $team->id,
+                    'database_uuid' => $database->uuid,
+                    'error' => $exception->getMessage(),
+                ]);
+            }
+        }
+
+        return ['restarted' => $restarted];
+    }
+
+    private function shouldRestart(Model $database): bool
+    {
+        $cached = $this->desiredRuntimeState->cachedDesired($database);
+
+        if ($cached === false) {
+            return false;
+        }
+
+        if ($cached === true) {
+            return true;
+        }
+
+        // Crash without prior desired flag: restart if a linked application is up.
+        return $this->hasRunningLinkedApplication($database);
+    }
+
+    private function hasRunningLinkedApplication(Model $database): bool
+    {
+        $comment = LibsqlConnectionEnvSync::LINK_COMMENT_PREFIX.$database->uuid;
+
+        $applicationIds = EnvironmentVariable::query()
+            ->where('is_preview', false)
+            ->where('comment', $comment)
+            ->where('resourceable_type', Application::class)
+            ->pluck('resourceable_id')
+            ->unique()
+            ->filter()
+            ->values();
+
+        if ($applicationIds->isEmpty()) {
+            return false;
+        }
+
+        return Application::query()
+            ->whereIn('id', $applicationIds)
+            ->get(['id', 'status'])
+            ->contains(fn (Application $application): bool => $this->isRunningStatus((string) ($application->status ?? ''))
+                || $this->isStartingStatus((string) ($application->status ?? '')));
+    }
+
+    private function isRunningStatus(string $status): bool
+    {
+        return str($status)->before(':')->lower()->trim()->value() === 'running';
+    }
+
+    private function isStartingStatus(string $status): bool
+    {
+        $primary = str($status)->before(':')->lower()->trim()->value();
+
+        return in_array($primary, ['starting', 'restarting', 'created'], true);
+    }
+
+    private function isStoppedStatus(string $status): bool
+    {
+        $primary = str($status)->before(':')->lower()->trim()->value();
+
+        return str($status)->startsWith('exited')
+            || in_array($primary, ['stopped', 'dead', 'exited'], true);
+    }
+}

backend/config/devforge.php

@@ -86,6 +86,21 @@
         'desired_ttl_seconds' => (int) env('DEVFORGE_APPLICATION_KEEP_ALIVE_TTL', 60 * 60 * 24 * 30),
     ],
 
+    /*
+    |--------------------------------------------------------------------------
+    | Keep-alive bases de données
+    |--------------------------------------------------------------------------
+    |
+    | Redémarre les bases standalone arrêtées involontairement (crash / host).
+    | Un stop manuel désactive le redémarrage auto jusqu’au prochain start.
+    | Les bases rattachées à une app encore en cours sont aussi relancées.
+    |
+    */
+    'database_keep_alive' => [
+        'enabled' => env('DEVFORGE_DATABASE_KEEP_ALIVE', true),
+        'desired_ttl_seconds' => (int) env('DEVFORGE_DATABASE_KEEP_ALIVE_TTL', 60 * 60 * 24 * 30),
+    ],
+
     'agents_monitor_build_enabled' => env(
         'DEVFORGE_AGENTS_MONITOR_BUILD_ENABLED',
         env('DEVFORGE_AGENTS_WEBHOOK_BUILD_ENABLED', true),