diff --git a/backend/app/Console/Kernel.php b/backend/app/Console/Kernel.php index 54e939e02d..fb9f6111e1 100644 --- a/backend/app/Console/Kernel.php +++ b/backend/app/Console/Kernel.php @@ -13,6 +13,7 @@ use App\Jobs\DevForge\ApplicationKeepAliveJob; use App\Jobs\DevForge\ApplicationReadinessWatchdogJob; use App\Jobs\DevForge\CheckDockerImageUpdatesJob; +use App\Jobs\DevForge\DatabaseKeepAliveJob; use App\Jobs\DevForge\InstanceHostDiskGuardJob; use App\Jobs\PullChangelog; use App\Jobs\PullTemplatesFromCDN; @@ -85,6 +86,10 @@ protected function schedule(Schedule $schedule): void ->everyTwoMinutes() ->onOneServer(); + $this->scheduleInstance->job(new DatabaseKeepAliveJob) + ->everyTwoMinutes() + ->onOneServer(); + $this->scheduleInstance->job(new InstanceHostDiskGuardJob) ->everyFiveMinutes() ->onOneServer(); @@ -132,6 +137,10 @@ protected function schedule(Schedule $schedule): void ->everyTwoMinutes() ->onOneServer(); + $this->scheduleInstance->job(new DatabaseKeepAliveJob) + ->everyTwoMinutes() + ->onOneServer(); + $this->scheduleInstance->job(new InstanceHostDiskGuardJob) ->everyFiveMinutes() ->onOneServer(); diff --git a/backend/app/Jobs/DevForge/DatabaseKeepAliveJob.php b/backend/app/Jobs/DevForge/DatabaseKeepAliveJob.php new file mode 100644 index 0000000000..e4f086f329 --- /dev/null +++ b/backend/app/Jobs/DevForge/DatabaseKeepAliveJob.php @@ -0,0 +1,29 @@ +onQueue('default'); + } + + public function handle(DatabaseKeepAliveService $keepAlive): void + { + $keepAlive->tickAllTeams(); + } +} diff --git a/backend/app/Services/DevForge/Core/CoreResourceAction.php b/backend/app/Services/DevForge/Core/CoreResourceAction.php index 1600f0426e..494696186b 100644 --- a/backend/app/Services/DevForge/Core/CoreResourceAction.php +++ b/backend/app/Services/DevForge/Core/CoreResourceAction.php @@ -12,6 +12,7 @@ use App\Models\Application; use App\Models\Service; use App\Services\DevForge\Application\ApplicationDesiredRuntimeState; +use App\Services\DevForge\Database\DatabaseDesiredRuntimeState; use Illuminate\Database\Eloquent\Model; use RuntimeException; use Symfony\Component\HttpKernel\Exception\HttpException; @@ -21,6 +22,7 @@ class CoreResourceAction { public function __construct( private readonly ApplicationDesiredRuntimeState $desiredRuntimeState, + private readonly DatabaseDesiredRuntimeState $databaseDesiredRuntimeState, ) {} /** @@ -117,6 +119,12 @@ private function database(Model $database, string $action, array $options): arra throw new HttpException(422, $exception->getMessage(), previous: $exception); } + if ($action === 'stop') { + $this->databaseDesiredRuntimeState->markDesiredStopped($database); + } else { + $this->databaseDesiredRuntimeState->markDesiredRunning($database); + } + return [ 'queued' => false, 'completed' => true, diff --git a/backend/app/Services/DevForge/Database/DatabaseDesiredRuntimeState.php b/backend/app/Services/DevForge/Database/DatabaseDesiredRuntimeState.php new file mode 100644 index 0000000000..a4ce26e7f4 --- /dev/null +++ b/backend/app/Services/DevForge/Database/DatabaseDesiredRuntimeState.php @@ -0,0 +1,77 @@ +cacheKey((string) $database->uuid), + true, + now()->addSeconds($this->ttlSeconds()), + ); + } + + public function markDesiredStopped(Model $database): void + { + Cache::put( + $this->cacheKey((string) $database->uuid), + false, + now()->addSeconds($this->ttlSeconds()), + ); + } + + /** + * Explicit cache value: true = must run, false = stopped on purpose, null = unknown. + */ + public function cachedDesired(Model $database): ?bool + { + $cached = Cache::get($this->cacheKey((string) $database->uuid)); + + if ($cached === true) { + return true; + } + + if ($cached === false) { + return false; + } + + return null; + } + + public function isDesiredRunning(Model $database): bool + { + $cached = $this->cachedDesired($database); + + if ($cached === true) { + return true; + } + + if ($cached === false) { + return false; + } + + return $this->isRunningStatus((string) ($database->status ?? '')); + } + + private function cacheKey(string $uuid): string + { + return self::CACHE_PREFIX.$uuid; + } + + private function ttlSeconds(): int + { + return max(3600, (int) config('devforge.database_keep_alive.desired_ttl_seconds', 60 * 60 * 24 * 30)); + } + + private function isRunningStatus(string $status): bool + { + return str($status)->before(':')->lower()->trim()->value() === 'running'; + } +} diff --git a/backend/app/Services/DevForge/Database/DatabaseKeepAliveService.php b/backend/app/Services/DevForge/Database/DatabaseKeepAliveService.php new file mode 100644 index 0000000000..871a685929 --- /dev/null +++ b/backend/app/Services/DevForge/Database/DatabaseKeepAliveService.php @@ -0,0 +1,160 @@ +enabled()) { + return; + } + + Team::query() + ->select(['id']) + ->orderBy('id') + ->each(function (Team $team): void { + try { + $this->tickTeam($team); + } catch (\Throwable $exception) { + Log::warning('DevForge database keep-alive tick failed.', [ + 'team_id' => $team->id, + 'error' => $exception->getMessage(), + ]); + } + }); + } + + /** + * @return array{restarted: array} + */ + public function tickTeam(Team $team): array + { + if (! $this->enabled()) { + return ['restarted' => []]; + } + + $databases = $this->catalog->resources($team, 'databases'); + if ($databases->isEmpty()) { + return ['restarted' => []]; + } + + $restarted = []; + + foreach ($databases as $database) { + if (! $database instanceof Model || blank($database->uuid)) { + continue; + } + + $status = (string) ($database->status ?? 'unknown'); + + if ($this->isRunningStatus($status) || $this->isStartingStatus($status)) { + $this->desiredRuntimeState->markDesiredRunning($database); + + continue; + } + + if (! $this->isStoppedStatus($status)) { + continue; + } + + if (! $this->shouldRestart($database)) { + continue; + } + + try { + if ($this->runtimeGuard->ensureRunning($database)) { + $this->desiredRuntimeState->markDesiredRunning($database); + $restarted[] = (string) $database->uuid; + } + } catch (\Throwable $exception) { + Log::warning('DevForge database keep-alive failed to start database.', [ + 'team_id' => $team->id, + 'database_uuid' => $database->uuid, + 'error' => $exception->getMessage(), + ]); + } + } + + return ['restarted' => $restarted]; + } + + private function shouldRestart(Model $database): bool + { + $cached = $this->desiredRuntimeState->cachedDesired($database); + + if ($cached === false) { + return false; + } + + if ($cached === true) { + return true; + } + + // Crash without prior desired flag: restart if a linked application is up. + return $this->hasRunningLinkedApplication($database); + } + + private function hasRunningLinkedApplication(Model $database): bool + { + $comment = LibsqlConnectionEnvSync::LINK_COMMENT_PREFIX.$database->uuid; + + $applicationIds = EnvironmentVariable::query() + ->where('is_preview', false) + ->where('comment', $comment) + ->where('resourceable_type', Application::class) + ->pluck('resourceable_id') + ->unique() + ->filter() + ->values(); + + if ($applicationIds->isEmpty()) { + return false; + } + + return Application::query() + ->whereIn('id', $applicationIds) + ->get(['id', 'status']) + ->contains(fn (Application $application): bool => $this->isRunningStatus((string) ($application->status ?? '')) + || $this->isStartingStatus((string) ($application->status ?? ''))); + } + + private function isRunningStatus(string $status): bool + { + return str($status)->before(':')->lower()->trim()->value() === 'running'; + } + + private function isStartingStatus(string $status): bool + { + $primary = str($status)->before(':')->lower()->trim()->value(); + + return in_array($primary, ['starting', 'restarting', 'created'], true); + } + + private function isStoppedStatus(string $status): bool + { + $primary = str($status)->before(':')->lower()->trim()->value(); + + return str($status)->startsWith('exited') + || in_array($primary, ['stopped', 'dead', 'exited'], true); + } +} diff --git a/backend/config/devforge.php b/backend/config/devforge.php index 274c945fdc..b6a121fe73 100644 --- a/backend/config/devforge.php +++ b/backend/config/devforge.php @@ -86,6 +86,21 @@ 'desired_ttl_seconds' => (int) env('DEVFORGE_APPLICATION_KEEP_ALIVE_TTL', 60 * 60 * 24 * 30), ], + /* + |-------------------------------------------------------------------------- + | Keep-alive bases de données + |-------------------------------------------------------------------------- + | + | Redémarre les bases standalone arrêtées involontairement (crash / host). + | Un stop manuel désactive le redémarrage auto jusqu’au prochain start. + | Les bases rattachées à une app encore en cours sont aussi relancées. + | + */ + 'database_keep_alive' => [ + 'enabled' => env('DEVFORGE_DATABASE_KEEP_ALIVE', true), + 'desired_ttl_seconds' => (int) env('DEVFORGE_DATABASE_KEEP_ALIVE_TTL', 60 * 60 * 24 * 30), + ], + 'agents_monitor_build_enabled' => env( 'DEVFORGE_AGENTS_MONITOR_BUILD_ENABLED', env('DEVFORGE_AGENTS_WEBHOOK_BUILD_ENABLED', true), diff --git a/backend/tests/Unit/DevForge/DatabaseKeepAliveServiceTest.php b/backend/tests/Unit/DevForge/DatabaseKeepAliveServiceTest.php new file mode 100644 index 0000000000..c67df5c255 --- /dev/null +++ b/backend/tests/Unit/DevForge/DatabaseKeepAliveServiceTest.php @@ -0,0 +1,94 @@ +set('devforge.enabled', true); + config()->set('devforge.database_keep_alive.enabled', true); + config()->set('devforge.database_keep_alive.desired_ttl_seconds', 3600); +}); + +function makeKeepAliveDatabase(string $uuid, string $status): StandaloneLibsql +{ + $database = Mockery::mock(StandaloneLibsql::class)->makePartial(); + $database->uuid = $uuid; + $database->status = $status; + $database->shouldReceive('getMorphClass')->andReturn(StandaloneLibsql::class); + + return $database; +} + +it('restarts desired stopped databases via the runtime guard', function () { + $team = Team::factory()->make(['id' => 31]); + $stopped = makeKeepAliveDatabase('db-keep-1', 'exited:unhealthy'); + + $desired = new DatabaseDesiredRuntimeState; + $desired->markDesiredRunning($stopped); + + $catalog = Mockery::mock(CoreResourceCatalog::class); + $catalog->shouldReceive('resources') + ->with($team, 'databases') + ->andReturn(new Collection([$stopped])); + + $guard = Mockery::mock(StandaloneDatabaseRuntimeGuard::class); + $guard->shouldReceive('ensureRunning') + ->once() + ->with($stopped) + ->andReturn(true); + + $keepAlive = new DatabaseKeepAliveService($catalog, $guard, $desired); + $result = $keepAlive->tickTeam($team); + + expect($result['restarted'])->toBe(['db-keep-1']); +}); + +it('does not restart a database stopped on purpose', function () { + $team = Team::factory()->make(['id' => 32]); + $stopped = makeKeepAliveDatabase('db-keep-2', 'exited:unhealthy'); + + $desired = new DatabaseDesiredRuntimeState; + $desired->markDesiredStopped($stopped); + + $catalog = Mockery::mock(CoreResourceCatalog::class); + $catalog->shouldReceive('resources') + ->with($team, 'databases') + ->andReturn(new Collection([$stopped])); + + $guard = Mockery::mock(StandaloneDatabaseRuntimeGuard::class); + $guard->shouldReceive('ensureRunning')->never(); + + $keepAlive = new DatabaseKeepAliveService($catalog, $guard, $desired); + $result = $keepAlive->tickTeam($team); + + expect($result['restarted'])->toBe([]); +}); + +it('marks running databases as desired running', function () { + $team = Team::factory()->make(['id' => 33]); + $running = makeKeepAliveDatabase('db-keep-3', 'running:healthy'); + + $desired = new DatabaseDesiredRuntimeState; + + $catalog = Mockery::mock(CoreResourceCatalog::class); + $catalog->shouldReceive('resources') + ->with($team, 'databases') + ->andReturn(new Collection([$running])); + + $guard = Mockery::mock(StandaloneDatabaseRuntimeGuard::class); + $guard->shouldReceive('ensureRunning')->never(); + + $keepAlive = new DatabaseKeepAliveService($catalog, $guard, $desired); + $keepAlive->tickTeam($team); + + expect($desired->isDesiredRunning($running))->toBeTrue(); +});