diff --git a/docs/entity-diagram.md b/docs/entity-diagram.md index b57c33e..8dd0842 100644 --- a/docs/entity-diagram.md +++ b/docs/entity-diagram.md @@ -25,6 +25,7 @@ erDiagram INTEGER exit_code BOOLEAN log_truncated BOOLEAN pinned + BOOLEAN cancel_requested OID log_oid TIMESTAMP started_at TIMESTAMP finished_at @@ -56,6 +57,6 @@ erDiagram * `PROJECT.owner_subject` and `BUILD.requested_by` hold the Keycloak subject (`sub`) of the user. * `BUILD.project_id` is a plain column, not a foreign key constraint. * `BUILD.logoid` (type `lo`, from the `lo` extension) references the build log, stored as one PostgreSQL large object. It is not mapped in the `Build` entity. The `lo_manage` trigger unlinks the large object when the build is deleted, so no logs are orphaned. -* `BUILD.status` is `RUNNING`, `PASSED` or `FAILED`. +* `BUILD.status` is `QUEUED`, `RUNNING`, `PASSED`, `FAILED` or `CANCELLED`. `BUILD.cancel_requested` is set on a running build until the runner has stopped it. * `BRANDING` and `SETTINGS` are singleton rows with `id = 1`. Until `SETTINGS` is first saved, its defaults apply. * Users, roles and credentials live in Keycloak and are not stored in the cisys database. diff --git a/docs/manual/en/03-roles.md b/docs/manual/en/03-roles.md index 254bea6..c6555e2 100644 --- a/docs/manual/en/03-roles.md +++ b/docs/manual/en/03-roles.md @@ -10,11 +10,11 @@ The **Member** role is the standard read-only role. Members can view public proj ## Lead -The **Lead** role is for people who run projects. Leads can do everything a member can, create projects, start manual builds on projects they can see, and pin builds of the projects they created. +The **Lead** role is for people who run projects. Leads can do everything a member can, create projects, start manual builds on projects they can see, cancel the builds they started, and pin and cancel builds of the projects they created. ## Admin -The **Admin** role manages the installation. Admins can see every project including private projects of other users, start and pin builds, use credential references for private Git repositories, manage the build images, change the branding, and change the settings such as build retention, the build queue, the hosts build steps may reach and the checkout limits. +The **Admin** role manages the installation. Admins can see every project including private projects of other users, start, pin and cancel builds, use credential references for private Git repositories, manage the build images, change the branding, and change the settings such as build retention, the build queue, the hosts build steps may reach and the checkout limits. ## Access summary @@ -27,6 +27,7 @@ The **Admin** role manages the installation. Admins can see every project includ | View build history and logs | yes | yes | yes | | Start a manual build | no | yes | yes | | Pin a build | no | own | yes | +| Cancel a build | no | own | yes | | Use a Git credential reference | no | no | yes | | Manage build images | no | no | yes | | Change branding | no | no | yes | diff --git a/docs/manual/en/05-builds.md b/docs/manual/en/05-builds.md index f34e2ba..8c704b1 100644 --- a/docs/manual/en/05-builds.md +++ b/docs/manual/en/05-builds.md @@ -28,7 +28,11 @@ A checkout is limited to `BUILD_MAX_CHECKOUT_MIB` (`cisys.build.max-checkout-mib If saving a completed result fails, the runner retains it in memory and retries on each dispatcher tick without rerunning the build. Its slot stays reserved until the result is saved. -On restart, interrupted `RUNNING` builds become `FAILED` and retain their partial logs. `QUEUED` builds remain queued and run when a slot is available. Startup also removes leftover `cisys-build-*` containers and images and `cisys-*` checkout directories from an unclean stop; a cleanup failure is logged and does not stop startup. Queued and running builds are protected from retention cleanup and prevent project deletion. This runner supports one backend instance; startup recovery assumes the previous instance has stopped. +On restart, interrupted `RUNNING` builds become `FAILED`, or `CANCELLED` when a cancellation was requested, and retain their partial logs. `QUEUED` builds remain queued and run when a slot is available. Startup also removes leftover `cisys-build-*` containers and images and `cisys-*` checkout directories from an unclean stop; a cleanup failure is logged and does not stop startup. Queued and running builds are protected from retention cleanup and prevent project deletion. This runner supports one backend instance; startup recovery assumes the previous instance has stopped. + +## Cancel a build + +Admins, the project's creator and whoever started a build can cancel it while it is `QUEUED` or `RUNNING` with the cancel button on its build page. A queued build becomes `CANCELLED` at once and never runs. For a running build, the runner stops the fetch or the image build within about a second, forcibly after ten more seconds, removes the image and records `CANCELLED` with `Build cancelled` at the end of the log. A cancellation that arrives while the build is finishing still wins. Cancelling a finished build returns HTTP 409. ## Workflow file @@ -36,7 +40,7 @@ The workflow file format, how the image is built, and its limits are described i ## What a build records -Each build stores the requested branch, the text of the workflow file it read (shown on the build page), the full name of the base image it found, the credential reference name if one was used (shown to admins only), the base image digest when the container runtime reports one, the commit ID, the exit code of the image build, which is that of the failing step (none when the build timed out or never ran), who started it, and when it started and finished; the duration is derived from these times. +Each build stores the requested branch, the text of the workflow file it read (shown on the build page), the full name of the base image it found, the credential reference name if one was used (shown to admins only), the base image digest when the container runtime reports one, the commit ID, the exit code of the image build, which is that of the failing step (none when the build timed out, was cancelled or never ran), who started it, and when it started and finished; the duration is derived from these times. ## Build logs @@ -54,10 +58,10 @@ The project's creator and admins can **Pin** a build on its build page, and **Un ## Build retention -Every hour, and a minute after startup, cisys deletes old builds of each project together with their logs. A finished, unpinned build is deleted when either +Every hour, and a minute after startup, cisys deletes old builds of each project together with their logs. A finished or cancelled, unpinned build is deleted when either * it finished more than the maximum age ago (default 30 days), or -* the project has more newer finished, unpinned builds than the maximum count (default 50). +* the project has more newer finished or cancelled, unpinned builds than the maximum count (default 50). Admins change both limits under **Settings** → **Retention**; `0` turns a limit off. Running and pinned builds are never deleted, and pinned builds do not count towards the limit. Deleted builds cannot be restored, so pin a build to keep it. `BUILD_RETENTION_EVERY` (default `1h`) changes how often the cleanup runs. diff --git a/docs/manual/en/72-architecture.md b/docs/manual/en/72-architecture.md index 57af711..cb67ad3 100644 --- a/docs/manual/en/72-architecture.md +++ b/docs/manual/en/72-architecture.md @@ -11,7 +11,7 @@ The sources are in `src/backend/main/java/ai/mnemosyne/cisys`. * The data model uses JPA entities backed by [Hibernate ORM Panache](https://quarkus.io/guides/hibernate-orm-panache): `Project`, `Build`, `Branding` and `Settings`. * HTTP endpoints use JAX-RS via [Quarkus REST](https://quarkus.io/guides/rest): `ProjectResource` (`/api/session`, `/api/projects`), `BuildResource` (`/api/projects/{projectId}/builds`), `BrandingResource` (`/api/branding`) and `SettingsResource` (`/api/settings`). * `BuildLogs` stores each build log as one PostgreSQL large object, using JDBC and the pgjdbc `LargeObjectManager` directly rather than Hibernate. Output is appended while the build runs and streamed back from `/api/projects/{projectId}/builds/{buildId}/log`, about 1 MB per short transaction, so a log is never held in memory whole. cisys supports PostgreSQL only. -* `BuildRunner` fetches repositories with [JGit](https://www.eclipse.org/jgit/) and runs builds through the Docker or Podman command line. It claims `QUEUED` builds in request order, up to the concurrency limit set under Settings or in configuration. +* `BuildRunner` fetches repositories with [JGit](https://www.eclipse.org/jgit/) and runs builds through the Docker or Podman command line. It claims `QUEUED` builds in request order, up to the concurrency limit set under Settings or in configuration. Each dispatcher tick also stops the running builds whose cancellation was requested. * `EgressProxy` is a per-build HTTPS `CONNECT` proxy on a Unix socket. Podman mounts the socket directory into each step, which still runs with `--network=none`; `socat` in the step forwards `127.0.0.1:3128` to it. The proxy allows only the hosts set under Settings → Network (`NetworkAllowlist`) on the configured ports, and never protected addresses. * `BuildRetention` is a [scheduled](https://quarkus.io/guides/scheduler) job that deletes each project's expired, unpinned builds in batches with one SQL statement per batch; the `lo_manage` trigger removes their logs. * Authentication uses [Quarkus OIDC](https://quarkus.io/guides/security-oidc-bearer-token-authentication) in `service` mode with Keycloak realm roles. @@ -51,6 +51,7 @@ erDiagram INTEGER exit_code BOOLEAN log_truncated BOOLEAN pinned + BOOLEAN cancel_requested OID log_oid TIMESTAMP started_at TIMESTAMP finished_at diff --git a/src/backend/main/java/ai/mnemosyne/cisys/Build.java b/src/backend/main/java/ai/mnemosyne/cisys/Build.java index 6dafc2a..59072ac 100644 --- a/src/backend/main/java/ai/mnemosyne/cisys/Build.java +++ b/src/backend/main/java/ai/mnemosyne/cisys/Build.java @@ -22,7 +22,8 @@ public enum Status { QUEUED, RUNNING, PASSED, - FAILED + FAILED, + CANCELLED } @Enumerated(EnumType.STRING) @@ -60,6 +61,10 @@ public enum Status { @Column(nullable = false) public boolean pinned; + /** Set on a running build; the runner stops it and records it as cancelled. */ + @Column(nullable = false) + public boolean cancelRequested; + public Instant startedAt; public Instant finishedAt; } diff --git a/src/backend/main/java/ai/mnemosyne/cisys/BuildResource.java b/src/backend/main/java/ai/mnemosyne/cisys/BuildResource.java index ca394d9..714e89e 100644 --- a/src/backend/main/java/ai/mnemosyne/cisys/BuildResource.java +++ b/src/backend/main/java/ai/mnemosyne/cisys/BuildResource.java @@ -29,6 +29,7 @@ public class BuildResource { @Inject BuildRunner runner; @Inject BuildLogs logs; @Inject RepositoryUrlValidator repositoryUrls; + @Inject ProjectAccess access; /** The image and the commands come from the branch's {@value Workflow#PATH}. */ public record RunRequest(String branch, String credentialRef) {} @@ -158,6 +159,26 @@ private BuildResponse setPinned(Long id, Long buildId, boolean pinned) { return response(b, true); } + /** Admins, the project's owner and whoever requested the build may cancel it. */ + @POST + @Path("/{buildId}/cancel") + public BuildResponse cancel(@PathParam("projectId") Long id, @PathParam("buildId") Long buildId) { + Project p = visible(id); + Build b = Build.findById(buildId); + if (b == null || !id.equals(b.projectId)) throw new NotFoundException(); + if (!access.mayCancel(p, b)) throw new ForbiddenException(); + if (!runner.cancel(buildId)) + throw new WebApplicationException( + jakarta.ws.rs.core.Response.status(409) + .type(MediaType.APPLICATION_JSON) + .entity(java.util.Map.of("error", "The build has already finished.")) + .build()); + return response( + io.quarkus.narayana.jta.QuarkusTransaction.requiringNew() + .call(() -> Build.findById(buildId)), + true); + } + @POST public BuildResponse start(@PathParam("projectId") Long id, RunRequest input) { Project p = visible(id); diff --git a/src/backend/main/java/ai/mnemosyne/cisys/BuildRetention.java b/src/backend/main/java/ai/mnemosyne/cisys/BuildRetention.java index e5d58e4..5ee18b5 100644 --- a/src/backend/main/java/ai/mnemosyne/cisys/BuildRetention.java +++ b/src/backend/main/java/ai/mnemosyne/cisys/BuildRetention.java @@ -26,11 +26,11 @@ public class BuildRetention { private static final String DELETE = """ - delete from build where status in ('PASSED', 'FAILED') and not pinned and id in ( + delete from build where status in ('PASSED', 'FAILED', 'CANCELLED') and not pinned and id in ( select id from ( select id, coalesce(finishedat, startedat) as at, row_number() over ( partition by projectid order by coalesce(finishedat, startedat) desc nulls last, id desc) as n - from build where status in ('PASSED', 'FAILED') and not pinned) eligible + from build where status in ('PASSED', 'FAILED', 'CANCELLED') and not pinned) eligible where n > :maxBuilds or at < :cutoff limit :batch) """; diff --git a/src/backend/main/java/ai/mnemosyne/cisys/BuildRunner.java b/src/backend/main/java/ai/mnemosyne/cisys/BuildRunner.java index daf382b..f40bc4e 100644 --- a/src/backend/main/java/ai/mnemosyne/cisys/BuildRunner.java +++ b/src/backend/main/java/ai/mnemosyne/cisys/BuildRunner.java @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.microprofile.config.inject.ConfigProperty; @@ -135,10 +136,13 @@ private synchronized void retryCompletions() { } } - /** Called inside a fresh transaction, also when retrying a failed commit. */ + /** + * Called inside a fresh transaction, also when retrying a failed commit. A cancellation requested + * meanwhile wins. + */ void persistCompletion(Completion r) { Build.update( - "status = ?1, revision = ?2, exitCode = ?3, imageDigest = ?4, logTruncated = ?5, finishedAt = ?6 where id = ?7 and status = ?8", + "status = case when cancelRequested = true then ?9 else ?1 end, revision = ?2, exitCode = ?3, imageDigest = ?4, logTruncated = ?5, finishedAt = ?6 where id = ?7 and status = ?8", r.status(), r.revision(), r.exitCode(), @@ -146,7 +150,93 @@ void persistCompletion(Completion r) { r.logTruncated(), r.finishedAt(), r.id(), - Build.Status.RUNNING); + Build.Status.RUNNING, + Build.Status.CANCELLED); + } + + /** A build this runner executes, with the container command currently running for it. */ + static final class Running { + volatile boolean cancelled; + volatile Process process; + } + + final Map running = new ConcurrentHashMap<>(); + + /** + * A queued build is cancelled at once; a running one is stopped by the runner executing it. False + * when it already finished. + */ + boolean cancel(Long id) { + return QuarkusTransaction.requiringNew() + .call(() -> { + lockQueue(); + if (Build.update( + "status = ?1, finishedAt = ?2 where id = ?3 and status = ?4", + Build.Status.CANCELLED, + Instant.now(), + id, + Build.Status.QUEUED) + == 1) return true; + return Build.update( + "cancelRequested = true where id = ?1 and status = ?2", id, Build.Status.RUNNING) + == 1; + }); + } + + /** Stops this runner's builds whose cancellation was requested through the API. */ + void stopCancelled() { + if (running.isEmpty()) return; + List ids = + QuarkusTransaction.requiringNew() + .call(() -> + Build.getEntityManager() + .createQuery( + "select b.id from Build b where b.status = ?1 and b.cancelRequested = true and b.id in ?2", + Long.class) + .setParameter(1, Build.Status.RUNNING) + .setParameter(2, List.copyOf(running.keySet())) + .getResultList()); + for (Long id : ids) { + Running r = running.get(id); + if (r != null && !r.cancelled) kill(r); + } + } + + /** + * The flag is set before the process is read, and {@link #started} does the reverse, so one of + * them always stops it. + */ + void kill(Running r) { + r.cancelled = true; + Process p = r.process; + if (p != null) terminate(p); + } + + void started(Running r, Process p) { + r.process = p; + if (r.cancelled) terminate(p); + } + + /** SIGTERM first so the runtime removes its build container, then forcibly. */ + private void terminate(Process p) { + p.destroy(); + Runnable force = + () -> { + try { + if (!p.waitFor(10, TimeUnit.SECONDS)) { + p.descendants().forEach(ProcessHandle::destroyForcibly); + p.destroyForcibly(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + try { + executor.submit(force); + } catch (RejectedExecutionException e) { + p.descendants().forEach(ProcessHandle::destroyForcibly); + p.destroyForcibly(); + } } private volatile boolean ready; @@ -165,17 +255,19 @@ void recover(@Observes StartupEvent event) { if (maxCheckoutMib < 1 || maxCheckoutFiles < 1) throw new IllegalArgumentException("Checkout size and file limits must be positive"); cleanupOrphans(); - List running = + List interrupted = QuarkusTransaction.requiringNew() - .call(() -> - Build.list("status", Build.Status.RUNNING).stream().map(b -> b.id).toList()); - for (Long id : running) { - Throwable failure = logs.appendLine(id, "Server restarted during build"); + .call(() -> Build.list("status", Build.Status.RUNNING)); + for (Build build : interrupted) { + Long id = build.id; + boolean cancelled = build.cancelRequested; + Throwable failure = + logs.appendLine(id, cancelled ? "Build cancelled" : "Server restarted during build"); if (failure != null) Log.errorf(failure, "Could not append to the log of build %d", id); QuarkusTransaction.requiringNew() .run(() -> { Build b = Build.findById(id); - b.status = Build.Status.FAILED; + b.status = cancelled ? Build.Status.CANCELLED : Build.Status.FAILED; b.finishedAt = Instant.now(); if (failure != null) b.logTruncated = true; }); @@ -350,6 +442,7 @@ synchronized void dispatch() { if (!ready || stopping) return; try { retryCompletions(); + stopCancelled(); Build b; while ((b = claim()) != null) { Build task = b; @@ -477,6 +570,8 @@ private void run(Long id, String url, BuildResource.RunRequest request) { Throwable captureError = null; BuildLogs.Writer log = null; EgressProxy proxy = null; + Running handle = new Running(); + running.put(id, handle); try { log = logs.open(id); // Checked again here because DNS can change after the project was saved. @@ -496,9 +591,17 @@ private void run(Long id, String url, BuildResource.RunRequest request) { Path context = Files.createDirectory(dir.resolve("context")); Path workspace = context.resolve(Workflow.WORKSPACE); revision = - Checkout.fetch(workspace, url, request.branch(), credentials, limits[0], limits[1]); + Checkout.fetch( + workspace, + url, + request.branch(), + credentials, + limits[0], + limits[1], + () -> handle.cancelled); // The workflow on the fetched branch describes the image. String text = Workflow.read(workspace); + if (handle.cancelled) throw new CancellationException(); if (text == null) throw new IllegalArgumentException("The branch has no " + Workflow.PATH); // Stored before parsing, so a build keeps the file that it ran or that failed validation. QuarkusTransaction.requiringNew() @@ -552,7 +655,8 @@ private void run(Long id, String url, BuildResource.RunRequest request) { if (proxy != null) build.add("--volume=" + proxy.directory() + ":" + Workflow.PROXY_DIR + ":ro,Z"); build.add(context.toString()); - Streamed result = stream(build, 600, log, logMaxBytes); + if (handle.cancelled) throw new CancellationException(); + Streamed result = stream(build, 600, log, logMaxBytes, p -> started(handle, p)); captureError = result.readerError(); if (result.truncated()) { truncated = true; @@ -564,31 +668,52 @@ private void run(Long id, String url, BuildResource.RunRequest request) { + " further bytes discarded]"); } digest = digest(from.id()); + if (handle.cancelled) throw new CancellationException(); if (result.timeout()) log.line("Build timed out"); else { // The image build stops at the first failing RUN and exits with its exit code. exit = result.code(); status = exit == 0 ? "PASSED" : "FAILED"; } - } catch (InterruptedException e) { - if (log != null) log.line("Build stopped: the server is shutting down"); - else captureError = e; - } catch (IllegalArgumentException e) { - if (log != null) log.line("Build failed: " + e.getMessage()); - else captureError = e; } catch (Exception e) { - if (log != null) - log.line( - stopping - ? "Build stopped: the server is shutting down" - : "Build failed during checkout or container execution (" - + e.getClass().getSimpleName() - + "). Check the runner configuration."); - else captureError = e; + // A failure caused by the cancellation is not reported as one. + if (!handle.cancelled) { + if (e instanceof InterruptedException) { + if (log != null) log.line("Build stopped: the server is shutting down"); + else captureError = e; + } else if (e instanceof IllegalArgumentException) { + if (log != null) log.line("Build failed: " + e.getMessage()); + else captureError = e; + } else if (log != null) + log.line( + stopping + ? "Build stopped: the server is shutting down" + : "Build failed during checkout or container execution (" + + e.getClass().getSimpleName() + + "). Check the runner configuration."); + else captureError = e; + } } finally { // A shutdown interrupts the build; the cleanup and the result must still run, so clear the // flag until done. boolean interrupted = Thread.interrupted(); + running.remove(id, handle); + // The request can arrive after the last dispatcher poll; persistCompletion also honours one + // that arrives after this read. + boolean cancelled = handle.cancelled; + if (!cancelled && running()) + try { + cancelled = + QuarkusTransaction.requiringNew() + .call(() -> Build.count("id = ?1 and cancelRequested = true", id) > 0); + } catch (RuntimeException e) { + Log.warnf(e, "Could not check whether build %d was cancelled", id); + } + if (cancelled) { + status = "CANCELLED"; + exit = null; + if (log != null) log.line("Build cancelled"); + } // The image only exists to run the workflow. try { command(List.of(runtime, "rmi", "-f", name), 60, 4096); @@ -771,7 +896,24 @@ record Streamed( static Streamed stream(List args, int timeout, LogSink sink, long maxBytes) throws Exception { - return stream(args, timeout, sink, maxBytes, 30_000, 5_000); + return stream(args, timeout, sink, maxBytes, p -> {}); + } + + static Streamed stream( + List args, int timeout, LogSink sink, long maxBytes, Consumer started) + throws Exception { + return stream(args, timeout, sink, maxBytes, 30_000, 5_000, started); + } + + static Streamed stream( + List args, + int timeout, + LogSink sink, + long maxBytes, + long drainMillis, + long forceMillis) + throws Exception { + return stream(args, timeout, sink, maxBytes, drainMillis, forceMillis, p -> {}); } /** @@ -784,10 +926,12 @@ static Streamed stream( LogSink sink, long maxBytes, long drainMillis, - long forceMillis) + long forceMillis, + Consumer started) throws Exception { Process p = new ProcessBuilder(args).redirectErrorStream(true).start(); p.getOutputStream().close(); + started.accept(p); InputStream raw = p.getInputStream(); AtomicLong total = new AtomicLong(); AtomicReference error = new AtomicReference<>(); diff --git a/src/backend/main/java/ai/mnemosyne/cisys/Checkout.java b/src/backend/main/java/ai/mnemosyne/cisys/Checkout.java index 92016d0..3be4db2 100644 --- a/src/backend/main/java/ai/mnemosyne/cisys/Checkout.java +++ b/src/backend/main/java/ai/mnemosyne/cisys/Checkout.java @@ -18,11 +18,13 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.function.BooleanSupplier; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.dircache.DirCacheCheckout; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.EmptyProgressMonitor; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.Repository; @@ -44,8 +46,20 @@ final class Checkout { private Checkout() {} + static String fetch( + Path dir, + String url, + String branch, + CredentialsProvider credentials, + int maxMib, + int maxFiles) + throws Exception { + return fetch(dir, url, branch, credentials, maxMib, maxFiles, () -> false); + } + /** * Returns the commit ID. {@code dir} must not exist; it is removed again when the checkout fails. + * The fetch stops once {@code cancelled} returns true. */ static String fetch( Path dir, @@ -53,7 +67,8 @@ static String fetch( String branch, CredentialsProvider credentials, int maxMib, - int maxFiles) + int maxFiles, + BooleanSupplier cancelled) throws Exception { String ref = Constants.R_REMOTES + "origin/" + branch; Files.createDirectory(dir); @@ -73,7 +88,14 @@ static String fetch( .setRemote(url) .setRefSpecs(new RefSpec("+refs/heads/" + branch + ":" + ref)) .setDepth(1) - .setTimeout(60); + .setTimeout(60) + .setProgressMonitor( + new EmptyProgressMonitor() { + @Override + public boolean isCancelled() { + return cancelled.getAsBoolean(); + } + }); if (credentials != null) fetch.setCredentialsProvider(credentials); try { fetch.call(); diff --git a/src/backend/main/java/ai/mnemosyne/cisys/ProjectAccess.java b/src/backend/main/java/ai/mnemosyne/cisys/ProjectAccess.java index 36df40f..feadc49 100644 --- a/src/backend/main/java/ai/mnemosyne/cisys/ProjectAccess.java +++ b/src/backend/main/java/ai/mnemosyne/cisys/ProjectAccess.java @@ -43,4 +43,18 @@ boolean canSeePrivate(Project p) { return false; return identity.hasRole("admin") || jwt.getSubject().equals(p.ownerSubject); } + + /** Whether the build is queued or running and {@link #mayCancel} allows it. */ + boolean canCancel(Project p, Build b) { + return (b.status == Build.Status.QUEUED || b.status == Build.Status.RUNNING) && mayCancel(p, b); + } + + /** Admins, the project's owner and whoever requested the build. */ + boolean mayCancel(Project p, Build b) { + if (canSeePrivate(p)) return true; + if (!(identity.getPrincipal() instanceof JsonWebToken jwt) || jwt.getSubject() == null) + return false; + return (identity.hasRole("lead") || identity.hasRole("admin")) + && jwt.getSubject().equals(b.requestedBy); + } } diff --git a/src/backend/main/java/ai/mnemosyne/cisys/ProjectPageResource.java b/src/backend/main/java/ai/mnemosyne/cisys/ProjectPageResource.java index 2aa27c8..e2396ed 100644 --- a/src/backend/main/java/ai/mnemosyne/cisys/ProjectPageResource.java +++ b/src/backend/main/java/ai/mnemosyne/cisys/ProjectPageResource.java @@ -63,6 +63,7 @@ public record PageBuild( boolean logTruncated, boolean pinned, boolean canPin, + boolean canCancel, @com.fasterxml.jackson.annotation.JsonInclude( com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL) String workflow) {} @@ -82,7 +83,8 @@ public ProjectPage page(@PathParam("name") String name, @QueryParam("branch") St .list(); // Whoever may see a private project may also pin its builds. boolean canPin = access.canSeePrivate(p); - List builds = rows.stream().map(b -> pageBuild(b, canPin, false)).toList(); + List builds = + rows.stream().map(b -> pageBuild(b, canPin, access.canCancel(p, b), false)).toList(); return new ProjectPage( new ProjectInfo(p.id, p.name, p.repositoryUrl, p.privateRepository), builds); } @@ -97,11 +99,12 @@ public PageBuild build(@PathParam("name") String name, @PathParam("buildId") Lon Project p = access.visible(name); Build b = Build.findById(buildId); if (b == null || !p.id.equals(b.projectId)) throw new NotFoundException(); - return pageBuild(b, access.canSeePrivate(p), true); + return pageBuild(b, access.canSeePrivate(p), access.canCancel(p, b), true); } /** The workflow file is sent for a single build only, as it can be up to 64 KiB. */ - private static PageBuild pageBuild(Build b, boolean canPin, boolean withWorkflow) { + private static PageBuild pageBuild( + Build b, boolean canPin, boolean canCancel, boolean withWorkflow) { return new PageBuild( b.id, b.projectId, @@ -116,6 +119,7 @@ private static PageBuild pageBuild(Build b, boolean canPin, boolean withWorkflow b.logTruncated, b.pinned, canPin, + canCancel, withWorkflow ? b.workflow : null); } diff --git a/src/backend/main/resources/db/migration/R__schema.sql b/src/backend/main/resources/db/migration/R__schema.sql index b37bc16..2d86bcb 100644 --- a/src/backend/main/resources/db/migration/R__schema.sql +++ b/src/backend/main/resources/db/migration/R__schema.sql @@ -104,3 +104,6 @@ ALTER TABLE settings ADD COLUMN IF NOT EXISTS networkallowlist text; -- Null uses the configured checkout limits. ALTER TABLE settings ADD COLUMN IF NOT EXISTS maxcheckoutmib integer; ALTER TABLE settings ADD COLUMN IF NOT EXISTS maxcheckoutfiles integer; + +-- Set on a running build until the runner has stopped it. +ALTER TABLE build ADD COLUMN IF NOT EXISTS cancelrequested boolean NOT NULL DEFAULT false; diff --git a/src/backend/test/java/ai/mnemosyne/cisys/BuildQueueTest.java b/src/backend/test/java/ai/mnemosyne/cisys/BuildQueueTest.java index 0106bac..72df57a 100644 --- a/src/backend/test/java/ai/mnemosyne/cisys/BuildQueueTest.java +++ b/src/backend/test/java/ai/mnemosyne/cisys/BuildQueueTest.java @@ -457,4 +457,137 @@ void retentionProtectsQueuedBuilds() { retention.purge(java.time.Instant.now(), 1, 1); assertEquals(Build.Status.QUEUED, read(waiting.id).status); } + + @Test + void cancelledQueuedBuildIsNeverClaimed() { + Build cancelled = submit(), next = submit(); + assertTrue(runner.cancel(cancelled.id)); + Build stored = read(cancelled.id); + assertEquals(Build.Status.CANCELLED, stored.status); + assertNotNull(stored.finishedAt); + assertEquals(next.id, runner.claim().id); + assertFalse(runner.cancel(cancelled.id)); + } + + @Test + void dispatcherKillsRunningBuildOnceItsCancellationIsRequested() { + List killed = new CopyOnWriteArrayList<>(); + BuildRunner worker = + new BuildRunner() { + @Override + void kill(Running r) { + killed.add(r); + r.cancelled = true; + } + }; + worker.logs = logs; + worker.concurrency = 1; + worker.queueLimit = 100; + worker.recover(null); + try { + Build b = submit(), other = submit(); + runner.concurrency = 2; + runner.claim(); + runner.claim(); + BuildRunner.Running handle = new BuildRunner.Running(), untouched = new BuildRunner.Running(); + worker.running.put(b.id, handle); + worker.running.put(other.id, untouched); + worker.dispatch(); + assertTrue(killed.isEmpty()); + assertTrue(runner.cancel(b.id)); + assertEquals(Build.Status.RUNNING, read(b.id).status); + assertTrue(read(b.id).cancelRequested); + worker.dispatch(); + assertEquals(List.of(handle), killed); + worker.dispatch(); + assertEquals(1, killed.size(), "a handle is killed once"); + } finally { + worker.stop(); + } + } + + @Test + void cancellationRequestedBeforeCompletionWins() { + Build b = submit(); + runner.claim(); + assertTrue(runner.cancel(b.id)); + runner.complete( + new BuildRunner.Completion( + b.id, Build.Status.PASSED, "revision", 0, null, false, java.time.Instant.now())); + Build stored = read(b.id); + assertEquals(Build.Status.CANCELLED, stored.status); + assertNotNull(stored.finishedAt); + assertFalse(runner.cancel(b.id)); + } + + @Test + void completionWithoutCancellationKeepsItsStatus() { + Build b = submit(); + runner.claim(); + runner.complete( + new BuildRunner.Completion( + b.id, Build.Status.FAILED, "revision", 1, null, false, java.time.Instant.now())); + assertEquals(Build.Status.FAILED, read(b.id).status); + } + + @Test + void processStartedAfterKillIsStopped() throws Exception { + BuildRunner.Running handle = new BuildRunner.Running(); + runner.kill(handle); + long start = System.nanoTime(); + var r = + BuildRunner.stream( + List.of("sleep", "60"), + 30, + (bytes, off, len) -> {}, + Long.MAX_VALUE, + p -> runner.started(handle, p)); + assertFalse(r.timeout()); + assertTrue( + System.nanoTime() - start < TimeUnit.SECONDS.toNanos(15), + "the process is stopped as soon as it starts"); + } + + @Test + void killStopsRunningProcess() throws Exception { + BuildRunner.Running handle = new BuildRunner.Running(); + CountDownLatch published = new CountDownLatch(1); + Thread stopper = + Thread.ofPlatform() + .start(() -> { + try { + if (published.await(10, TimeUnit.SECONDS)) runner.kill(handle); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + long start = System.nanoTime(); + var r = + BuildRunner.stream( + List.of("sleep", "60"), + 30, + (bytes, off, len) -> {}, + Long.MAX_VALUE, + p -> { + runner.started(handle, p); + published.countDown(); + }); + stopper.join(); + assertFalse(r.timeout()); + assertTrue(System.nanoTime() - start < TimeUnit.SECONDS.toNanos(15)); + } + + @Test + void retentionPurgesCancelledBuilds() { + Build b = submit(); + assertTrue(runner.cancel(b.id)); + retention.purge(java.time.Instant.now(), 0, 1); + assertNotNull(read(b.id), "the newest build is kept"); + submit(); + Build newer = submit(); + assertTrue(runner.cancel(newer.id)); + retention.purge(java.time.Instant.now(), 0, 1); + assertNull(read(b.id)); + assertNotNull(read(newer.id)); + } } diff --git a/src/backend/test/java/ai/mnemosyne/cisys/BuildResourceTest.java b/src/backend/test/java/ai/mnemosyne/cisys/BuildResourceTest.java index d73c36d..2976b11 100644 --- a/src/backend/test/java/ai/mnemosyne/cisys/BuildResourceTest.java +++ b/src/backend/test/java/ai/mnemosyne/cisys/BuildResourceTest.java @@ -522,4 +522,99 @@ void leadDoesNotSeeRequester() { void adminSeesRequester() { requesterVisible(true); } + + private static Build stored(Long build) { + return QuarkusTransaction.requiringNew().call(() -> Build.findById(build)); + } + + private static void requestedBy(Long build, String subject) { + QuarkusTransaction.requiringNew() + .run(() -> Build.update("requestedBy = ?1 where id = ?2", subject, build)); + } + + private static String cancel(Long project, Long build) { + return "/api/projects/" + project + "/builds/" + build + "/cancel"; + } + + @Test + @TestSecurity(user = "lead", roles = "lead") + @JwtSecurity(claims = @Claim(key = "sub", value = "owner-id")) + void ownerCancelsQueuedAndRunningBuilds() { + Long id = project(false); + Long queued = build(id, "QUEUED"), running = build(id, "RUNNING"), done = build(id, "PASSED"); + given() + .contentType("application/json") + .post(cancel(id, queued)) + .then() + .statusCode(200) + .body("status", equalTo("CANCELLED"), "finishedAt", notNullValue()); + given() + .contentType("application/json") + .post(cancel(id, running)) + .then() + .statusCode(200) + .body("status", equalTo("RUNNING")); + assertTrue(stored(running).cancelRequested); + given() + .contentType("application/json") + .post(cancel(id, done)) + .then() + .statusCode(409) + .body("error", equalTo("The build has already finished.")); + assertEquals(Build.Status.PASSED, stored(done).status); + } + + @Test + @TestSecurity(user = "admin", roles = "admin") + @JwtSecurity(claims = @Claim(key = "sub", value = "admin-id")) + void adminCancelsPrivateBuildOfOthers() { + Long id = project(true); + Long b = build(id, "RUNNING"); + given().contentType("application/json").post(cancel(id, b)).then().statusCode(200); + assertTrue(stored(b).cancelRequested); + } + + @Test + @TestSecurity(user = "lead", roles = "lead") + @JwtSecurity(claims = @Claim(key = "sub", value = "requester-id")) + void requesterCancelsOnlyOwnBuild() { + Long id = project(false); + Long own = build(id, "RUNNING"), others = build(id, "RUNNING"); + requestedBy(own, "requester-id"); + given().contentType("application/json").post(cancel(id, own)).then().statusCode(200); + assertTrue(stored(own).cancelRequested); + given().contentType("application/json").post(cancel(id, others)).then().statusCode(403); + assertFalse(stored(others).cancelRequested); + } + + @Test + @TestSecurity(user = "member", roles = "member") + @JwtSecurity(claims = @Claim(key = "sub", value = "other-id")) + void memberCannotCancel() { + Long id = project(false); + Long b = build(id, "RUNNING"); + given().contentType("application/json").post(cancel(id, b)).then().statusCode(403); + assertFalse(stored(b).cancelRequested); + } + + @Test + @TestSecurity(user = "lead", roles = "lead") + @JwtSecurity(claims = @Claim(key = "sub", value = "owner-id")) + void cancelOfBuildInOtherProjectIs404() { + Long id = project(false), other = project(false); + Long b = build(other, "RUNNING"); + given().contentType("application/json").post(cancel(id, b)).then().statusCode(404); + assertFalse(stored(b).cancelRequested); + } + + @Test + void recoverCancelsRequestedBuild() { + Long id = project(false); + Long b = build(id, "RUNNING", "partial\n"); + QuarkusTransaction.requiringNew() + .run(() -> Build.update("cancelRequested = true where id = ?1", b)); + runner.recover(null); + assertEquals(Build.Status.CANCELLED, stored(b).status); + assertEquals("partial\nBuild cancelled\n", logs.readAll(b)); + } } diff --git a/src/backend/test/java/ai/mnemosyne/cisys/ProjectPageResourceTest.java b/src/backend/test/java/ai/mnemosyne/cisys/ProjectPageResourceTest.java index 8f133bb..2bbaab7 100644 --- a/src/backend/test/java/ai/mnemosyne/cisys/ProjectPageResourceTest.java +++ b/src/backend/test/java/ai/mnemosyne/cisys/ProjectPageResourceTest.java @@ -416,4 +416,46 @@ void adminSeesPrivateProject() { .statusCode(200) .body("builds[0].canPin", equalTo(true)); } + + private void setStatus(long buildId, Build.Status status) { + QuarkusTransaction.requiringNew() + .run(() -> Build.update("status = ?1 where id = ?2", status, buildId)); + } + + @Test + @TestSecurity(user = "lead", roles = "lead") + @JwtSecurity(claims = @Claim(key = "sub", value = "owner-id")) + void ownerMayCancelOnlyActiveBuilds() { + Seeded s = seed(false); + String path = "/api/project-pages/" + s.name() + "/builds/" + s.buildId(); + given().get(path).then().statusCode(200).body("canCancel", equalTo(false)); + setStatus(s.buildId(), Build.Status.QUEUED); + try { + given().get(path).then().statusCode(200).body("canCancel", equalTo(true)); + given() + .get("/api/project-pages/" + s.name()) + .then() + .statusCode(200) + .body("builds[0].canCancel", equalTo(true)); + } finally { + setStatus(s.buildId(), Build.Status.CANCELLED); + } + } + + @Test + @TestSecurity(user = "lead", roles = "lead") + @JwtSecurity(claims = @Claim(key = "sub", value = "other-id")) + void otherLeadCannotCancel() { + Seeded s = seed(false); + setStatus(s.buildId(), Build.Status.RUNNING); + try { + given() + .get("/api/project-pages/" + s.name() + "/builds/" + s.buildId()) + .then() + .statusCode(200) + .body("canCancel", equalTo(false)); + } finally { + setStatus(s.buildId(), Build.Status.FAILED); + } + } } diff --git a/src/frontend/src/BuildStatus.jsx b/src/frontend/src/BuildStatus.jsx index fbf9acf..b0029ee 100644 --- a/src/frontend/src/BuildStatus.jsx +++ b/src/frontend/src/BuildStatus.jsx @@ -7,13 +7,21 @@ */ import React from "react"; -import { Circle, Clock, RefreshCw, ThumbsDown, ThumbsUp } from "lucide-react"; +import { + Ban, + Circle, + Clock, + RefreshCw, + ThumbsDown, + ThumbsUp, +} from "lucide-react"; const LABELS = { PASSED: "Passed", FAILED: "Failed", RUNNING: "Running", QUEUED: "Queued", + CANCELLED: "Cancelled", }; /** Thumbs up for a passing build, thumbs down for a failing one, a spinner while it runs. */ @@ -41,6 +49,13 @@ export default function BuildStatus({ status }) { ); } + if (status === "CANCELLED") { + return ( + + + ); + } if (status === "RUNNING") { return ( diff --git a/src/frontend/src/ProjectBuild.jsx b/src/frontend/src/ProjectBuild.jsx index f0d72ff..32e554f 100644 --- a/src/frontend/src/ProjectBuild.jsx +++ b/src/frontend/src/ProjectBuild.jsx @@ -7,7 +7,7 @@ */ import React, { useCallback, useEffect, useState } from "react"; -import { ArrowLeft, Pin, PinOff } from "lucide-react"; +import { ArrowLeft, Pin, PinOff, Square } from "lucide-react"; import { Button } from "./components/ui/button"; import { Card, @@ -35,6 +35,7 @@ export default function ProjectBuild({ const [build, setBuild] = useState(undefined); const [error, setError] = useState(""); const [pinning, setPinning] = useState(false); + const [cancelling, setCancelling] = useState(false); const loadBuild = useCallback(async () => { try { @@ -65,6 +66,21 @@ export default function ProjectBuild({ } } + async function cancel() { + setCancelling(true); + setError(""); + try { + await load(`/projects/${build.projectId}/builds/${build.id}/cancel`, { + method: "POST", + }); + await loadBuild(); + } catch (e) { + setError(e.message || "Unable to cancel the build."); + } finally { + setCancelling(false); + } + } + useEffect(() => { const initial = setTimeout(loadBuild, 0); const timer = setInterval(loadBuild, 5000); @@ -100,6 +116,19 @@ export default function ProjectBuild({ title={`Build #${build.id}`} actions={ <> + {build.canCancel && ( + + )} {build.canPin && (