From a89dbd42762b05eae6f7325031591fdb03ae05cc Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Tue, 17 Nov 2026 01:10:12 +0100 Subject: [PATCH 1/3] feat(proto): add STDIN_TOO_LARGE proof request error --- crates/types/network/src/types.rs | 4 ++++ proto/types.proto | 2 ++ 2 files changed, 6 insertions(+) diff --git a/crates/types/network/src/types.rs b/crates/types/network/src/types.rs index 49fc893..9287de9 100644 --- a/crates/types/network/src/types.rs +++ b/crates/types/network/src/types.rs @@ -3074,6 +3074,8 @@ pub enum ProofRequestError { ProgramTooLarge = 7, /// The program artifact cannot be decoded. InvalidProgram = 8, + /// The stdin artifact exceeds the size limit. + StdinTooLarge = 9, } impl ProofRequestError { /// String value of the enum field names used in the ProtoBuf definition. @@ -3091,6 +3093,7 @@ impl ProofRequestError { Self::ProvingFailure => "PROVING_FAILURE", Self::ProgramTooLarge => "PROGRAM_TOO_LARGE", Self::InvalidProgram => "INVALID_PROGRAM", + Self::StdinTooLarge => "STDIN_TOO_LARGE", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -3107,6 +3110,7 @@ impl ProofRequestError { "PROVING_FAILURE" => Some(Self::ProvingFailure), "PROGRAM_TOO_LARGE" => Some(Self::ProgramTooLarge), "INVALID_PROGRAM" => Some(Self::InvalidProgram), + "STDIN_TOO_LARGE" => Some(Self::StdinTooLarge), _ => None, } } diff --git a/proto/types.proto b/proto/types.proto index 0bde8e8..4ee5acf 100644 --- a/proto/types.proto +++ b/proto/types.proto @@ -2003,4 +2003,6 @@ enum ProofRequestError { PROGRAM_TOO_LARGE = 7; // The program artifact cannot be decoded. INVALID_PROGRAM = 8; + // The stdin artifact exceeds the size limit. + STDIN_TOO_LARGE = 9; } \ No newline at end of file From d4cde152b1b63fce25955ee4c3741eba4c334502 Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Tue, 17 Nov 2026 01:16:21 +0100 Subject: [PATCH 2/3] feat(artifacts): cap the size of a parallel download --- Cargo.lock | 1 + crates/network/artifacts/Cargo.toml | 1 + crates/network/artifacts/src/lib.rs | 98 +++++++++++++++++++++++++---- 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e76374..f7d338a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7534,6 +7534,7 @@ dependencies = [ "reqwest", "serde", "spn-artifact-types", + "thiserror 1.0.69", "tokio", "tracing", "url", diff --git a/crates/network/artifacts/Cargo.toml b/crates/network/artifacts/Cargo.toml index 8c84509..e504538 100644 --- a/crates/network/artifacts/Cargo.toml +++ b/crates/network/artifacts/Cargo.toml @@ -24,6 +24,7 @@ bytes = { workspace = true } lazy_static = { workspace = true } prost = { workspace = true } serde = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/crates/network/artifacts/src/lib.rs b/crates/network/artifacts/src/lib.rs index b5dd829..4a8805d 100644 --- a/crates/network/artifacts/src/lib.rs +++ b/crates/network/artifacts/src/lib.rs @@ -116,7 +116,7 @@ impl Artifact { let mut delay = Duration::from_secs(1); // Start with 1 second delay loop { - match download_s3_file(&s3_client, s3_bucket, &self.id, artifact_type).await { + match download_s3_file(&s3_client, s3_bucket, &self.id, artifact_type, u64::MAX).await { Ok(bytes) => return Ok(bytes), Err(e) => { retry_count += 1; @@ -163,9 +163,9 @@ impl Artifact { let bucket = parsed_url.host_str().ok_or_else(|| anyhow!("S3 URI missing bucket: {uri}"))?; let s3_client = get_s3_client(s3_region).await; - download_s3_file(&s3_client, bucket, &self.id, artifact_type).await + download_s3_file(&s3_client, bucket, &self.id, artifact_type, u64::MAX).await } - "https" => download_https_file(uri).await, + "https" => download_https_file(uri, u64::MAX).await, scheme => Err(anyhow!("Unsupported URI scheme for download_raw_from_uri: {scheme}")), } } @@ -182,6 +182,7 @@ impl Artifact { /// * `s3_region` - The AWS region for S3 operations /// * `artifact_type` - The type of artifact determining the S3 prefix /// * `concurrency` - Optional concurrency limit (default: 8) + /// * `max_bytes` - The largest artifact to accept; a larger one fails with [`ArtifactTooLarge`] #[instrument(fields(label = self.label, id = self.id), skip_all)] pub async fn download_raw_from_uri_par( &self, @@ -189,6 +190,7 @@ impl Artifact { s3_region: &str, artifact_type: ArtifactType, concurrency: Option, + max_bytes: u64, ) -> Result { let parsed_url = Url::parse(uri).context("Failed to parse URI")?; match parsed_url.scheme() { @@ -196,9 +198,17 @@ impl Artifact { let bucket = parsed_url.host_str().ok_or_else(|| anyhow!("S3 URI missing bucket: {uri}"))?; let s3_client = get_s3_client(s3_region).await; - download_s3_file_par(&s3_client, bucket, &self.id, artifact_type, concurrency).await + download_s3_file_par( + &s3_client, + bucket, + &self.id, + artifact_type, + concurrency, + max_bytes, + ) + .await } - "https" => download_https_file_par(uri, concurrency).await, + "https" => download_https_file_par(uri, concurrency, max_bytes).await, scheme => { Err(anyhow!("Unsupported URI scheme for download_raw_from_uri_par: {scheme}")) } @@ -452,11 +462,20 @@ async fn get_s3_client(s3_region: &str) -> Arc { } } +/// A download refused because the artifact is larger than the caller accepts. +#[derive(Debug, thiserror::Error)] +#[error("artifact exceeds {max_bytes} bytes")] +pub struct ArtifactTooLarge { + /// The size limit of the refused download. + pub max_bytes: u64, +} + async fn download_s3_file( client: &S3Client, bucket: &str, id: &str, artifact_type: ArtifactType, + max_bytes: u64, ) -> Result { let key = get_s3_key(artifact_type, id); @@ -468,6 +487,10 @@ async fn download_s3_file( .await .context("Failed to get object from S3")?; + if res.content_length().is_some_and(|len| len as u64 > max_bytes) { + return Err(ArtifactTooLarge { max_bytes }.into()); + } + let data = res.body.collect().await.context("Failed to read S3 object body")?; let bytes = data.into_bytes(); @@ -480,6 +503,7 @@ async fn download_s3_file_par( id: &str, artifact_type: ArtifactType, concurrency: Option, + max_bytes: u64, ) -> Result { let key = get_s3_key(artifact_type, id); let concurrency = concurrency.unwrap_or(DEFAULT_CONCURRENCY); @@ -498,8 +522,12 @@ async fn download_s3_file_par( return Err(anyhow!("Invalid object size: {size}")); } + if size as u64 > max_bytes { + return Err(ArtifactTooLarge { max_bytes }.into()); + } + if size as usize <= CHUNK_SIZE { - return download_s3_file(client, bucket, id, artifact_type).await; + return download_s3_file(client, bucket, id, artifact_type, max_bytes).await; } let starts: Vec<(usize, i64)> = (0..size).step_by(CHUNK_SIZE).enumerate().collect(); @@ -575,9 +603,9 @@ async fn download_s3_file_par( Ok(Bytes::from(result)) } -async fn download_https_file(uri: &str) -> Result { +async fn download_https_file(uri: &str, max_bytes: u64) -> Result { let client = reqwest::Client::new(); - let res = client + let mut res = client .get(uri) .timeout(Duration::from_secs(90)) .send() @@ -586,12 +614,25 @@ async fn download_https_file(uri: &str) -> Result { if !res.status().is_success() { return Err(anyhow!("Failed to download from HTTPS URL {uri}: status {}", res.status())); } - let bytes = res.bytes().await.context("Failed to read HTTPS response body")?; - Ok(bytes) + if res.content_length().is_some_and(|len| len > max_bytes) { + return Err(ArtifactTooLarge { max_bytes }.into()); + } + let mut body = Vec::new(); + while let Some(chunk) = res.chunk().await.context("Failed to read HTTPS response body")? { + if (body.len() + chunk.len()) as u64 > max_bytes { + return Err(ArtifactTooLarge { max_bytes }.into()); + } + body.extend_from_slice(&chunk); + } + Ok(body.into()) } #[allow(clippy::too_many_lines)] -async fn download_https_file_par(uri: &str, concurrency: Option) -> Result { +async fn download_https_file_par( + uri: &str, + concurrency: Option, + max_bytes: u64, +) -> Result { let concurrency = concurrency.unwrap_or(DEFAULT_CONCURRENCY); let client = reqwest::Client::new(); @@ -622,13 +663,17 @@ async fn download_https_file_par(uri: &str, concurrency: Option) -> Resul .and_then(|v| v.parse::().ok()); if !supports_range || size.is_none() { - return download_https_file(uri).await; + return download_https_file(uri, max_bytes).await; } let size = size.unwrap(); + if size as u64 > max_bytes { + return Err(ArtifactTooLarge { max_bytes }.into()); + } + if size <= CHUNK_SIZE { - return download_https_file(uri).await; + return download_https_file(uri, max_bytes).await; } let starts: Vec<(usize, usize)> = (0..size).step_by(CHUNK_SIZE).enumerate().collect(); @@ -751,9 +796,36 @@ async fn upload_file( mod tests { use super::*; use spn_artifact_types::ArtifactType; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[test] fn private_stdin_has_its_own_prefix() { assert_eq!(get_s3_prefix(ArtifactType::PrivateStdin), "private-stdins"); } + + async fn serve(response: &'static str) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let uri = format!("http://{}/", listener.local_addr().unwrap()); + tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let _ = socket.read(&mut [0; 1024]).await.unwrap(); + socket.write_all(response.as_bytes()).await.unwrap(); + } + }); + uri + } + + #[tokio::test] + async fn https_download_rejects_a_body_over_the_cap() { + let with_length = "HTTP/1.1 200 OK\r\ncontent-length: 10\r\n\r\n0123456789"; + let chunked = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\na\r\n0123456789\r\n0\r\n\r\n"; + for response in [with_length, chunked] { + let uri = serve(response).await; + assert_eq!(download_https_file(&uri, 10).await.unwrap().len(), 10); + let err = download_https_file(&uri, 9).await.unwrap_err(); + assert!(err.is::()); + } + } } From 2c0489bc6a153edc4a513858390b7e6092b34610 Mon Sep 17 00:00:00 2001 From: Tobias Bergkvist Date: Tue, 17 Nov 2026 01:29:55 +0100 Subject: [PATCH 3/3] feat(artifacts): cap the size of a download from a URI --- crates/network/artifacts/src/lib.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/network/artifacts/src/lib.rs b/crates/network/artifacts/src/lib.rs index 4a8805d..5c7be0c 100644 --- a/crates/network/artifacts/src/lib.rs +++ b/crates/network/artifacts/src/lib.rs @@ -150,12 +150,14 @@ impl Artifact { /// * `uri` - The URI to download from (s3:// or https://) /// * `s3_region` - The AWS region for S3 operations /// * `artifact_type` - The type of artifact determining the S3 prefix + /// * `max_bytes` - The largest artifact to accept; a larger one fails with [`ArtifactTooLarge`] #[instrument(fields(label = self.label, id = self.id), skip_all)] pub async fn download_raw_from_uri( &self, uri: &str, s3_region: &str, artifact_type: ArtifactType, + max_bytes: u64, ) -> Result { let parsed_url = Url::parse(uri).context("Failed to parse URI")?; match parsed_url.scheme() { @@ -163,9 +165,9 @@ impl Artifact { let bucket = parsed_url.host_str().ok_or_else(|| anyhow!("S3 URI missing bucket: {uri}"))?; let s3_client = get_s3_client(s3_region).await; - download_s3_file(&s3_client, bucket, &self.id, artifact_type, u64::MAX).await + download_s3_file(&s3_client, bucket, &self.id, artifact_type, max_bytes).await } - "https" => download_https_file(uri, u64::MAX).await, + "https" => download_https_file(uri, max_bytes).await, scheme => Err(anyhow!("Unsupported URI scheme for download_raw_from_uri: {scheme}")), } } @@ -245,7 +247,8 @@ impl Artifact { uri: &str, s3_region: &str, ) -> Result { - let bytes = self.download_raw_from_uri(uri, s3_region, ArtifactType::Program).await?; + let bytes = + self.download_raw_from_uri(uri, s3_region, ArtifactType::Program, u64::MAX).await?; bincode::deserialize(&bytes).context("Failed to deserialize program from URI") } @@ -279,7 +282,8 @@ impl Artifact { uri: &str, s3_region: &str, ) -> Result { - let bytes = self.download_raw_from_uri(uri, s3_region, ArtifactType::Stdin).await?; + let bytes = + self.download_raw_from_uri(uri, s3_region, ArtifactType::Stdin, u64::MAX).await?; bincode::deserialize(&bytes).context("Failed to deserialize stdin from URI") } @@ -313,7 +317,8 @@ impl Artifact { uri: &str, s3_region: &str, ) -> Result { - let bytes = self.download_raw_from_uri(uri, s3_region, ArtifactType::Proof).await?; + let bytes = + self.download_raw_from_uri(uri, s3_region, ArtifactType::Proof, u64::MAX).await?; bincode::deserialize(&bytes).context("Failed to deserialize proof from URI") }