diff --git a/README.queued-deployments.md b/README.queued-deployments.md new file mode 100644 index 0000000..d48db7d --- /dev/null +++ b/README.queued-deployments.md @@ -0,0 +1,207 @@ +# 📦 Queued Deployments Guide + +This document explains the new Redis-backed queuing behavior for deployments in `vps-deployster`, and how to use it from CI (e.g., GitHub Actions) or any client. + +## What changed? + +Previously, when a deployment was already running for a project, a new request to `POST /deploy` returned HTTP 409: + +``` +Project has another deployment activity in progress. +``` + +Now, deployment requests are queued per project: + +- If a project’s deployment lock is available, the deployment starts immediately and you get `{ job_id }`. +- If a project’s deployment lock is held by another job, your request still returns HTTP 200 with: + +```json +{ "job_id": "", "queued": true, "message": "Another deployment is running; this job is queued." } +``` + +The server internally waits until the lock is free, then starts your deployment. No restart or re-run is required. + +## How it works + +- A Redis lock key `lock:deploy:` serializes deployments per project. +- When the lock is not available, the server: + - Returns `{ job_id, queued: true }` immediately. + - Polls for the lock in the background (every ~3s). + - Starts the queued job automatically once the lock is acquired. +- Logs and status are streamed to Redis during execution. +- On completion (success or failure), the deployment is recorded in the DB and the lock is released. + +> Default lock TTL is 600 seconds. If your builds routinely exceed this, consider increasing the TTL or adding a lock “watchdog” to refresh expirations during long runs. + +## API usage + +### POST /deploy + +Headers: + +``` +x-deployster-token: +Content-Type: application/json +``` + +Body: + +```json +{ + "cd": "/var/www/my-app", + "commands": [ + "yarn install --frozen-lockfile", + "yarn build", + "supervisorctl restart my-app--main" + ], + "commit_hash": "", + "ref_name": "main" +} +``` + +Responses: + +``` +200 { "job_id": "" } +200 { "job_id": "", "queued": true, "message": "Another deployment is running; this job is queued." } +403 { "error": "Unauthorized" } +``` + +Important rules: + +- The last command in `commands` must be `supervisorctl start|restart ` — otherwise the server fails the job early for safety. +- If your project uses pipeline stages configured in `pipeline_json`, the server will inject a `.env` file based on the stage’s environment variables before running your commands. + +### GET /status/:job_id + +``` +200 { "status": "queued|running|complete|failed|not_found", "logs": "..." } +404 { "status": "not_found" } +``` + +Notes: + +- Logs are appended as the job runs. +- Each call clears the returned chunk of logs from Redis. Keep polling to stream logs in near real-time. + +## GitHub Actions example + +Use GitHub’s `concurrency` to limit overlapping runs (optional), and poll `/status/:job_id` until completion. + +```yaml +name: Deploy + +on: + push: + branches: ["main"] + +# Queue workflows at the GitHub level (optional but recommended) +concurrency: + group: deploy-${{ github.ref }} + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Trigger deploy + id: trigger + run: | + RESPONSE=$(curl -sS -X POST \ + -H "x-deployster-token: ${{ secrets.DEPLOYSTER_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d @- "${{ secrets.DEPLOYSTER_URL }}/deploy" <<'JSON' + { + "cd": "/var/www/my-app", + "commands": [ + "yarn install --frozen-lockfile", + "yarn build", + "supervisorctl restart my-app--main" + ], + "commit_hash": "${{ github.sha }}", + "ref_name": "${{ github.ref_name }}" + } + JSON + ) + echo "$RESPONSE" + JOB_ID=$(echo "$RESPONSE" | jq -r '.job_id') + if [ -z "$JOB_ID" ] || [ "$JOB_ID" = "null" ]; then + echo "Failed to obtain job_id" >&2 + exit 1 + fi + echo "job_id=$JOB_ID" >> "$GITHUB_OUTPUT" + + - name: Stream deploy status + run: | + JOB_ID="${{ steps.trigger.outputs.job_id }}" + echo "Waiting on job: $JOB_ID" + while true; do + RESP=$(curl -sS -H "x-deployster-token: ${{ secrets.DEPLOYSTER_TOKEN }}" \ + "${{ secrets.DEPLOYSTER_URL }}/status/$JOB_ID") || true + STATUS=$(echo "$RESP" | jq -r '.status // "not_found"') + LOGS=$(echo "$RESP" | jq -r '.logs // ""') + echo -n "$LOGS" + if [ "$STATUS" = "complete" ]; then + echo "\nDeployment complete" + break + elif [ "$STATUS" = "failed" ]; then + echo "\nDeployment failed" >&2 + exit 1 + elif [ "$STATUS" = "not_found" ]; then + echo "\nJob not found" >&2 + exit 1 + fi + sleep 3 + done +``` + +## Local curl example + +```bash +# Trigger deploy +curl -sS -X POST \ + -H "x-deployster-token: $DEPLOYSTER_TOKEN" \ + -H "Content-Type: application/json" \ + -d @- "$DEPLOYSTER_URL/deploy" <<'JSON' | tee /tmp/deploy-response.json +{ + "cd": "/var/www/my-app", + "commands": [ + "yarn install --frozen-lockfile", + "yarn build", + "supervisorctl restart my-app--main" + ], + "commit_hash": "$(git rev-parse HEAD)", + "ref_name": "main" +} +JSON + +# Extract job_id +JOB_ID=$(jq -r '.job_id' /tmp/deploy-response.json) + +# Poll status +while true; do + RESP=$(curl -sS -H "x-deployster-token: $DEPLOYSTER_TOKEN" "$DEPLOYSTER_URL/status/$JOB_ID") || true + STATUS=$(echo "$RESP" | jq -r '.status // "not_found"') + LOGS=$(echo "$RESP" | jq -r '.logs // ""') + echo -n "$LOGS" + if [ "$STATUS" = "complete" ]; then + echo "\nDeployment complete" + break + elif [ "$STATUS" = "failed" ]; then + echo "\nDeployment failed" >&2 + exit 1 + elif [ "$STATUS" = "not_found" ]; then + echo "\nJob not found" >&2 + exit 1 + fi + sleep 3 +done +``` + +## Tips + +- Ensure your supervisor program name in the final command matches your configuration (e.g., `my-app--main`). +- Keep `DEPLOYSTER_TOKEN` secret and send it via the `x-deployster-token` header. +- If your builds are long, consider increasing the lock TTL beyond 600s or implementing a lock extension watchdog. diff --git a/app.js b/app.js index aa63e06..921cb08 100644 --- a/app.js +++ b/app.js @@ -83,6 +83,173 @@ app.post("/deploy", async (req, res) => { const job_id = Date.now().toString(); const { cd, commands, commit_hash, ref_name } = req.body; + // Helper to start the deployment (used for immediate and queued runs) + async function performDeployment() { + try { + // CREATE DEPLOYMENT RECORD LOG + try { + deploymentRecord = await createRowAndReturn( + "deployments", + "INSERT INTO deployments (user_id, project_id, commit_hash, pipeline_stage_uuid, action, status, started_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + user.id, + projectInView.id, + commit_hash, + pipelineJSON + ? pipelineJSON.filter((pipeline) => pipeline.git_branch == ref_name)[0]?.stage_uuid + : null, + "DEPLOY", + "RUNNING", + deploymentTimestamp, + ] + ); + } catch (error) { + console.error("Error recording deployment log:", error); + } + + await redisClient.set(`job:${job_id}:status`, "running"); + + // VALIDATE LAST COMMAND + const localCommands = [...commands]; + const lastCommand = localCommands[localCommands.length - 1] || ""; + if (!/supervisorctl\s+(start|restart)\s+[\w\-]+/.test(lastCommand)) { + const errorOutput = + "[ERROR] Last command must be a 'supervisorctl start|restart'. Aborting...\n"; + await redisClient.set(`job:${job_id}:status`, "failed"); + await redisClient.append(`job:${job_id}:logs`, errorOutput); + if (deploymentRecord) { + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + } + return; + } + + // INJECT ENV ENVIRONMENT + if (pipelineJSON && Array.isArray(pipelineJSON)) { + const pipelineStage = pipelineJSON.find((p) => p.git_branch === ref_name); + + if (pipelineStage && Array.isArray(pipelineStage.environment_variables)) { + const keyValues = pipelineStage.environment_variables.map((env) => { + const key = env.key ?? env.KEY; + const value = env.value ?? env.VALUE; + return `${key}=${value}`; + }); + + const envString = keyValues.join("\n"); + const escapedEnvString = envString.replace(/"/g, '\\"'); + const envCommand = `echo "${escapedEnvString}" > .env`; + localCommands.unshift(envCommand); + } + } + + // Append additional supervisor commands + const rereadCommand = `supervisorctl reread`; + const updateCommand = `supervisorctl update`; + localCommands.splice(localCommands.length - 1, 0, rereadCommand, updateCommand); + + const fullDeploymentCommandList = [...gitCommands, ...localCommands]; + + for (let i = 0; i < fullDeploymentCommandList.length; i++) { + const command = `cd ${cd} && ${fullDeploymentCommandList[i]}`; + const formattedCommand = `\n[${i + 1}] $ ${fullDeploymentCommandList[i]}\n`; + await redisClient.append(`job:${job_id}:logs`, formattedCommand); + + try { + const output = await runShell(command); + await redisClient.append(`job:${job_id}:logs`, output); + if (deploymentRecord) + await addLogToDeploymentRecord(deploymentRecord.id, output); + } catch (err) { + const errorOutput = `[ERROR] ${err}\n`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + if (deploymentRecord) { + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + } + return; + } + } + + await redisClient.set(`job:${job_id}:status`, "complete"); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.COMPLETED, + null, + null // HOLD ON SUCCESSFUL LOCK-KEY for a moment. It's passed a little later + ); + + if (pipelineJSON) { + var newLogMessage = await updatePipelineGitHead( + projectInView.id, + ref_name, + commit_hash + ); + await addLogToDeploymentRecord( + deploymentRecord.id, + `\n${newLogMessage}\n` + ); + } + + // STORE ARTIFACT + var newLogMessage = "\nCompressing artifact\n"; + await redisClient.set(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + const cleanCdPath = cd + .replace(/^\/*|\/*$/g, "") + .replace(/[^\w\-\/]/g, "_") + .replace(/\//g, "_"); + const artifactBase = path.join(__dirname, "deploy-artifacts"); + const artifactDir = path.join(artifactBase, cleanCdPath); + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const shortcommitHash = commit_hash.slice(0, 8); + const artifactPath = path.join( + artifactDir, + `deploy-${timestamp}-${shortcommitHash}.tar.gz` + ); + + try { + var newLogMessage = await runShell(`mkdir -p ${artifactDir}`); + await redisClient.set(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + + var newLogMessage = await runShell( + `tar -czf ${artifactPath} --exclude=".git" -C "${cd}" .` + ); + await redisClient.set(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + + var newLogMessage = `\n[INFO] Artifact created at ${artifactPath}\n`; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.COMPLETED, + artifactPath, + deploymentLockKey, + { createActivityLog: false } + ); + } catch (err) { + var newLogMessage = `[WARN] Artifact generation failed: ${err.message}\n`; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + } + } catch (e) { + console.error("Deployment error:", e); + } + } + if (!commit_hash) { // FOR version 1.0 BACKWARD COMPACTIBILITY await redisClient.set(`job:${job_id}:status`, "queued"); @@ -155,9 +322,26 @@ app.post("/deploy", async (req, res) => { ); if (!acquired) { - return res.status(409).json({ - error: "Project has another deployment activity in progress.", - }); + // Queue this deployment; respond immediately and start polling for the lock + await redisClient.set(`job:${job_id}:status`, "queued"); + await redisClient.del(`job:${job_id}:logs`); + res.json({ job_id, queued: true, message: "Another deployment is running; this job is queued." }); + + (async () => { + while (true) { + const got = await redisClient.set( + deploymentLockKey, + "locked", + "NX", + "EX", + 600 + ); + if (got) break; + await new Promise((r) => setTimeout(r, 3000)); + } + await performDeployment(); + })(); + return; } } catch (error) { throw Error("Error getting deployment lock key"); @@ -187,182 +371,15 @@ app.post("/deploy", async (req, res) => { console.error("Error processing project metadata:", error); } - try { - // CREATE DEPLOYMENT RECORD LOG - deploymentRecord = await createRowAndReturn( - "deployments", - "INSERT INTO deployments (user_id, project_id, commit_hash, pipeline_stage_uuid, action, status, started_at) VALUES (?, ?, ?, ?, ?, ?, ?)", - [ - user.id, - projectInView.id, - commit_hash, - pipelineJSON - ? pipelineJSON.filter( - (pipeline) => pipeline.git_branch == ref_name - )[0]?.stage_uuid - : null, - "DEPLOY", - "RUNNING", - deploymentTimestamp, - ] - ); - } catch (error) { - console.error("Error recording deployment log:", error); - } + // Deployment record is created when the job actually starts running await redisClient.set(`job:${job_id}:status`, "queued"); await redisClient.del(`job:${job_id}:logs`); res.json({ job_id }); - (async () => { - await redisClient.set(`job:${job_id}:status`, "running"); - - // VALIDATE LAST COMMAND - // Check that the last command includes supervisorctl start or supervisorctl restart: - const lastCommand = commands[commands.length - 1] || ""; - if (!/supervisorctl\s+(start|restart)\s+[\w\-]+/.test(lastCommand)) { - const errorOutput = - "[ERROR] Last command must be a 'supervisorctl start|restart'. Aborting...\n"; - await redisClient.set(`job:${job_id}:status`, "failed"); - await redisClient.append(`job:${job_id}:logs`, errorOutput); - if (deploymentRecord) { - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey - ); - } - return; - } - - // INJECT ENV ENVIRONMENT - if (pipelineJSON && Array.isArray(pipelineJSON)) { - const pipelineStage = pipelineJSON.find((p) => p.git_branch === ref_name); - - if (pipelineStage && Array.isArray(pipelineStage.environment_variables)) { - const keyValues = pipelineStage.environment_variables.map((env) => { - // SUPPORT lower and upper case - const key = env.key ?? env.KEY; - const value = env.value ?? env.VALUE; - return `${key}=${value}`; - }); - - const envString = keyValues.join("\n"); - const escapedEnvString = envString.replace(/"/g, '\\"'); - const envCommand = `echo "${escapedEnvString}" > .env`; - - // Insert .env creation right at the beginning of the command chain - commands.unshift(envCommand); - } - } - - // Append additional supervisor commands - const rereadCommand = `supervisorctl reread`; - const updateCommand = `supervisorctl update`; - - // Insert to the end of the last command - commands.splice(commands.length - 1, 0, rereadCommand, updateCommand); - - const fullDeploymentCommandList = [...gitCommands, ...commands]; - - for (let i = 0; i < fullDeploymentCommandList.length; i++) { - const command = `cd ${cd} && ${fullDeploymentCommandList[i]}`; - const formattedCommand = `\n[${i + 1}] $ ${ - fullDeploymentCommandList[i] - }\n`; - await redisClient.append(`job:${job_id}:logs`, formattedCommand); - - try { - const output = await runShell(command); - await redisClient.append(`job:${job_id}:logs`, output); - if (deploymentRecord) - await addLogToDeploymentRecord(deploymentRecord.id, output); - } catch (err) { - const errorOutput = `[ERROR] ${err}\n`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - if (deploymentRecord) { - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey - ); - } - return; - } - } - await redisClient.set(`job:${job_id}:status`, "complete"); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.COMPLETED, - null, - null // HOLD ON SUCCESSFUL LOCK-KEY for a moment. It's passed a little later - ); - - if (pipelineJSON) { - var newLogMessage = await updatePipelineGitHead( - projectInView.id, - ref_name, - commit_hash - ); - await addLogToDeploymentRecord( - deploymentRecord.id, - `\n${newLogMessage}\n` - ); - } - - // STORE ARTIFACT - var newLogMessage = "\nCompressing artifact\n"; - await redisClient.set(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - const cleanCdPath = cd - .replace(/^\/*|\/*$/g, "") // Remove leading/trailing slashes - .replace(/[^\w\-\/]/g, "_") // Replace non-word, non-hyphen, non-slash characters with underscore _ - .replace(/\//g, "_"); // Replace slash with underscore _ - const artifactBase = path.join(__dirname, "deploy-artifacts"); - const artifactDir = path.join(artifactBase, cleanCdPath); - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const shortcommitHash = commit_hash.slice(0, 8); - const artifactPath = path.join( - artifactDir, - `deploy-${timestamp}-${shortcommitHash}.tar.gz` - ); - - try { - // await fs.mkdir(artifactDir, { recursive: true }); - var newLogMessage = await runShell(`mkdir -p ${artifactDir}`); - await redisClient.set(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - - // Create archive -- excluding git to prevent future conflict - var newLogMessage = await runShell( - `tar -czf ${artifactPath} --exclude=".git" -C "${cd}" .` - ); - await redisClient.set(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - - var newLogMessage = `\n[INFO] Artifact created at ${artifactPath}\n`; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.COMPLETED, - artifactPath, - deploymentLockKey, - { createActivityLog: false } //skip Log regeneration - ); - } catch (err) { - var newLogMessage = `[WARN] Artifact generation failed: ${err.message}\n`; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - } - })(); + // Start deployment in background for immediate (non-queued) case + performDeployment(); }); app.get("/status/:job_id", async (req, res) => { diff --git a/controllers/index.js b/controllers/index.js index 9a21af0..31f87c3 100644 --- a/controllers/index.js +++ b/controllers/index.js @@ -1,4 +1,5 @@ const fs = require("fs").promises; +const fsSync = require("fs"); const path = require("path"); const net = require("net"); const moment = require("moment"); @@ -9,6 +10,7 @@ const { pool, getSingleRow, RecordDoesNotExist, + createRowAndReturn, } = require("@anclatechs/sql-buns"); const { createUserValidationSchema, @@ -41,6 +43,7 @@ const { runShell, startRedisServer, killRedisServer, + updatePipelineGitHead, } = require("../utils/functools"); const { DEPLOYMENT_STATUS } = require("../utils/constants"); const jwtr = new JWTR(redisClient); @@ -1203,94 +1206,117 @@ async function rollbackToCommitSnapshot(req, res) { const deploymentTimestamp = moment().format("YYYY-MM-DD HH:mm:ss"); const job_id = Date.now().toString(); - try { - projectInView = await getSingleRow( - ` - SELECT * - FROM projects - WHERE user_id = ? AND id = ? - `, - [user.id, project_id] - ); - } catch (error) { - return res - .status(403) - .json({ status: false, message: "Error getting project" }); - } + async function performRollback() { + try { + await redisClient.set(`job:${job_id}:status`, "running"); + + // CREATE DEPLOYMENT RECORD LOG + try { + deploymentRecord = await createRowAndReturn( + "deployments", + "INSERT INTO deployments (user_id, project_id, commit_hash, pipeline_stage_uuid, action, status, started_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + user.id, + projectInView.id, + commit_hash, + stage_uuid ?? null, + "ROLLBACK", + "RUNNING", + deploymentTimestamp, + ] + ); + var newLogMessage = "\n[INFO] Deployment record created\n"; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + } catch (error) { + var errorOutput = `[ERROR] Error recording deployment log: ${error}`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + if (deploymentRecord) await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord?.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + return; + } - try { - // GENERATE DEPLOYMENT LOCK KEY - deploymentLockKey = `lock:deploy:${projectInView.id}`; - const acquired = await redisClient.set( - deploymentLockKey, - "locked", - "NX", - "EX", - 600 // 10 mins - ); + if (stage_uuid) { + const pipelineJson = getProjectPipelineJSON(projectInView.pipeline_json); + if (pipelineJson) { + pipelineStageInView = pipelineJson.find((stage) => { + const { stage_uuid } = stage; + return pipelineJson.find( + (existingStage) => existingStage.stage_uuid === String(stage_uuid) + ); + }); + + if (!pipelineStageInView) { + var errorOutput = `[ERROR] Unable to reconcile stage UUID - ${stage_uuid}\n`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + return; + } - if (!acquired) { - return res.status(409).json({ - error: "Project has another deployment activity in progress.", - }); - } - } catch (error) { - return res - .status(403) - .json({ status: false, message: "Error getting deployment lock key" }); - } + var newLogMessage = `\n[INFO] Stage UUID - ${stage_uuid} processing\n`; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + } else { + var errorOutput = `[ERROR] Project has no active pipeline record\n`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + return; + } - await redisClient.set(`job:${job_id}:status`, "queued"); - await redisClient.del(`job:${job_id}:logs`); + PIPELINE_SQL_DEPLOYMENT_FILTER = `AND pipeline_stage_uuid = '${stage_uuid}'`; + } - try { - await redisClient.set(`job:${job_id}:status`, "running"); + var newLogMessage = `\n[INFO] Getting specific deployment record\n`; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - // CREATE DEPLOYMENT RECORD LOG - deploymentRecord = await createRowAndReturn( - "deployments", - "INSERT INTO deployments (user_id, project_id, commit_hash, pipeline_stage_uuid, action, status, started_at) VALUES (?, ?, ?, ?, ?, ?, ?)", - [ - user.id, - projectInView.id, - commit_hash, - stage_uuid ?? null, - "ROLLBACK", - "RUNNING", - deploymentTimestamp, - ] - ); - var newLogMessage = "\n[INFO] Deployment record created\n"; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - } catch (error) { - var errorOutput = `[ERROR] Error recording deployment log: ${error}`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey - ); - return res - .status(403) - .json({ status: false, message: "Error recording deployment log" }); - } + try { + specificDeploymentInView = await getSingleRow( + `SELECT * FROM deployments WHERE project_id = ? AND commit_hash = ? AND status = ? AND action = ? ${PIPELINE_SQL_DEPLOYMENT_FILTER}`, + [projectInView.id, commit_hash, "COMPLETED", "DEPLOY"] + ); - if (stage_uuid) { - const pipelineJson = getProjectPipelineJSON(projectInView.pipeline_json); - if (pipelineJson) { - pipelineStageInView = pipelineJson.find((stage) => { - const { stage_uuid } = stage; - return pipelineJson.find( - (existingStage) => existingStage.stage_uuid === String(stage_uuid) + var newLogMessage = `\n[INFO] Deployment record - ${specificDeploymentInView.id} obtained\n`; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + } catch (err) { + var errorOutput = `[ERROR] Error getting deployment record: ${err}\n`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey ); - }); + return; + } - if (!pipelineStageInView) { - var errorOutput = `[ERROR] Unable to reconcile stage UUID - ${stage_uuid}\n`; + if (specificDeploymentInView.artifact_path) { + backupFileLocation = specificDeploymentInView.artifact_path; + } else { + var errorOutput = `[ERROR] Error getting project/pipeline artifact path\n`; await redisClient.append(`job:${job_id}:logs`, errorOutput); await redisClient.set(`job:${job_id}:status`, "failed"); await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); @@ -1300,202 +1326,193 @@ async function rollbackToCommitSnapshot(req, res) { null, deploymentLockKey ); - return res.status(403).json({ - status: false, - message: "Unable to reconcile stage UUID", - }); + return; } - var newLogMessage = `\n[INFO] Stage UUID - ${stage_uuid} processing\n`; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - } else { - var errorOutput = `[ERROR] Project has no active pipeline record\n`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey - ); - return res.status(403).json({ - status: false, - message: "Project has no active pipeline record", - }); - } + // Reset Git HEAD + try { + var newLogMessage = await runShell( + `cd ${projectInView.app_local_path} && git reset --hard ${commit_hash}` + ); - PIPELINE_SQL_DEPLOYMENT_FILTER = `AND pipeline_stage_uuid = '${stage_uuid}'`; - } + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + } catch (err) { + var errorOutput = `[ERROR] ${err}\n`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + return; + } - var newLogMessage = `\n[INFO] Getting specific deployment record\n`; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + // Clean up files except .git + const files = fsSync.readdirSync(projectInView.app_local_path); + for (const file of files) { + if (![".git"].includes(file)) { + const fullPath = path.join(projectInView.app_local_path, file); + fsSync.rmSync(fullPath, { recursive: true, force: true }); + } + } - try { - specificDeploymentInView = await getSingleRow( - `SELECT * FROM deployments WHERE project_id = ? AND commit_hash = ? AND status = ? AND action = ? ${PIPELINE_SQL_DEPLOYMENT_FILTER}`, - [projectInView.id, commit_hash, "COMPLETED", "DEPLOY"] - ); + // Restore files from the backup + try { + var newLogMessage = await runShell( + `tar -xzf "${backupFileLocation}" -C "${projectInView.app_local_path}"` + ); + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + } catch (err) { + var errorOutput = `[ERROR] ${err}\n`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + return; + } - var newLogMessage = `\n[INFO] Deployment record - ${specificDeploymentInView.id} obtained\n`; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - } catch (err) { - var errorOutput = `[ERROR] Error getting deployment record: ${err}\n`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey - ); - return res - .status(403) - .json({ status: false, message: "Error getting deployment record" }); - } + try { + // STOP server + serverActionHandler( + projectInView.id, + "kill", + pipelineStageInView?.stage_uuid + ); - if (specificDeploymentInView.artifact_path) { - backupFileLocation = specificDeploymentInView.artifact_path; - } else { - var errorOutput = `[ERROR] Error getting project/pipeline artifact path\n`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey - ); + var newLogMessage = `\n[INFO] Server stopped && rebooting\n`; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - return res - .status(403) - .json({ status: false, message: "Error getting artifact path" }); - } + // REDEPLOY + serverActionHandler( + projectInView.id, + "redeploy", + pipelineStageInView?.stage_uuid + ); - // Reset Git HEAD - try { - var newLogMessage = await runShell( - `cd ${projectInView.app_local_path} && git reset --hard ${commit_hash}` - ); + var newLogMessage = `\n[INFO] Server restarted successfully\n`; + await redisClient.append(`job:${job_id}:logs`, newLogMessage); + await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); + } catch (err) { + var errorOutput = `[ERROR] ${err}\n`; + await redisClient.append(`job:${job_id}:logs`, errorOutput); + await redisClient.set(`job:${job_id}:status`, "failed"); + await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.FAILED, + null, + deploymentLockKey + ); + return; + } - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - } catch (err) { - var errorOutput = `[ERROR] ${err}\n`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey - ); - } + if (pipelineStageInView) { + var newLogMessage = await updatePipelineGitHead( + projectInView.id, + pipelineStageInView.git_branch, + commit_hash + ); + await addLogToDeploymentRecord( + deploymentRecord.id, + `\n${newLogMessage}\n` + ); + } else { + await pool.run("UPDATE projects SET current_head = ? WHERE id = ?", [ + commit_hash, + projectInView.id, + ]); + } + + await redisClient.set(`job:${job_id}:status`, "complete"); + + // Clear the logs from Redis upon completion + await redisClient.del(`job:${job_id}:logs`); - // Clean up files except .git - const files = fs.readdirSync(projectInView.app_local_path); - for (const file of files) { - if (![".git"].includes(file)) { - const fullPath = path.join(projectInView.app_local_path, file); - fs.rmSync(fullPath, { recursive: true, force: true }); + await markDeploymentAsComplete( + deploymentRecord.id, + DEPLOYMENT_STATUS.COMPLETED, + null, + deploymentLockKey + ); + } catch (e) { + console.error("Rollback error:", e); } } - // Restore files from the backup try { - var newLogMessage = await runShell( - `tar -xzf "${backupFileLocation}" -C "${projectInView.app_local_path}"` - ); - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - } catch (err) { - var errorOutput = `[ERROR] ${err}\n`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey + projectInView = await getSingleRow( + ` + SELECT * + FROM projects + WHERE user_id = ? AND id = ? + `, + [user.id, project_id] ); + } catch (error) { + return res + .status(403) + .json({ status: false, message: "Error getting project" }); } try { - // STOP server - serverActionHandler( - projectInView.id, - "kill", - pipelineStageInView?.stage_uuid - ); - - var newLogMessage = `\n[INFO] Server stopped && rebooting\n`; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - - // REDEPLOY - serverActionHandler( - projectInView.id, - "redeploy", - pipelineStageInView?.stage_uuid - ); - - var newLogMessage = `\n[INFO] Server restarted successfully\n`; - await redisClient.append(`job:${job_id}:logs`, newLogMessage); - await addLogToDeploymentRecord(deploymentRecord.id, newLogMessage); - } catch (err) { - var errorOutput = `[ERROR] ${err}\n`; - await redisClient.append(`job:${job_id}:logs`, errorOutput); - await redisClient.set(`job:${job_id}:status`, "failed"); - await addLogToDeploymentRecord(deploymentRecord.id, errorOutput); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.FAILED, - null, - deploymentLockKey + // GENERATE DEPLOYMENT LOCK KEY + deploymentLockKey = `lock:deploy:${projectInView.id}`; + const acquired = await redisClient.set( + deploymentLockKey, + "locked", + "NX", + "EX", + 600 // 10 mins ); - } - if (pipelineStageInView) { - var newLogMessage = await updatePipelineGitHead( - projectInView.id, - pipelineStageInView.git_branch, - commit_hash - ); - await addLogToDeploymentRecord( - deploymentRecord.id, - `\n${newLogMessage}\n` - ); - } else { - await pool.run("UPDATE projects SET current_head = ? WHERE id = ?", [ - commit_hash, - projectInView.id, - ]); + if (!acquired) { + // Queue this rollback; respond and poll for lock + await redisClient.set(`job:${job_id}:status`, "queued"); + await redisClient.del(`job:${job_id}:logs`); + res.json({ job_id, queued: true, message: "Another deployment is running; this job is queued." }); + + (async () => { + while (true) { + const got = await redisClient.set( + deploymentLockKey, + "locked", + "NX", + "EX", + 600 + ); + if (got) break; + await new Promise((r) => setTimeout(r, 3000)); + } + await performRollback(); + })(); + return; + } + } catch (error) { + return res + .status(403) + .json({ status: false, message: "Error getting deployment lock key" }); } - - await redisClient.set(`job:${job_id}:status`, "complete"); - - // Clear the logs from Redis upon completion + // Respond immediately and run rollback in background + await redisClient.set(`job:${job_id}:status`, "queued"); await redisClient.del(`job:${job_id}:logs`); - await markDeploymentAsComplete( - deploymentRecord.id, - DEPLOYMENT_STATUS.COMPLETED, - null, - deploymentLockKey - ); + res.json({ job_id }); - return res.json({ - status: true, - message: "Project details updated successfully", - data: {}, - }); + (async () => { + await performRollback(); + })(); + return; } catch (error) { console.log({ error }); return res