From 6fb5cf7a6c3366f8ecc61bcc75705ad6a1f4b14a Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Mon, 26 Jan 2026 04:18:19 +0200 Subject: [PATCH] Add /submitartifacts multipart log upload endpoint Add new endpoint to accept artifacts uploads via multipart and store them under /artifacts. Signed-off-by: Denys Fedoryshchenko --- kcidb-restd-rs/Cargo.toml | 2 +- kcidb-restd-rs/README.md | 77 +++++++++ kcidb-restd-rs/src/main.rs | 316 ++++++++++++++++++++++++++++++++++++- 3 files changed, 392 insertions(+), 3 deletions(-) create mode 100644 kcidb-restd-rs/README.md diff --git a/kcidb-restd-rs/Cargo.toml b/kcidb-restd-rs/Cargo.toml index ea9ed9e..dcb6654 100644 --- a/kcidb-restd-rs/Cargo.toml +++ b/kcidb-restd-rs/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] -axum = { version = "0.8.3", features = ["tokio"] } +axum = { version = "0.8.3", features = ["tokio", "multipart"] } axum-server = { version = "0.7.2", features = ["tls-rustls"] } clap = { version = "4.5.35", features = ["derive"] } jsonwebtoken = "9.3.1" diff --git a/kcidb-restd-rs/README.md b/kcidb-restd-rs/README.md new file mode 100644 index 0000000..d84b2d9 --- /dev/null +++ b/kcidb-restd-rs/README.md @@ -0,0 +1,77 @@ +# kcidb-restd-rs + +Simple REST receiver for KCIDB submissions and log uploads. + +## Endpoints + +### `POST /submit` +Accepts a JSON body and writes it to the spool directory as `submission-.json`. +Authentication uses the JWT `origin` claim. + +Example: +```bash +curl -X POST http://localhost:8080/submit \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + --data-binary @submission.json +``` + +### `POST /submitartifacts` +Accepts an artifact file via `multipart/form-data` and writes it to +`/artifacts/` with a filename: + +`__` + +The `origin` is taken from the JWT. The `filename` is taken from the uploaded +file's multipart filename. + +Required multipart fields: +- `submission_id` (string, `^[a-z0-9_]+$`, max 64 chars) +- `artifact` (file upload; multipart filename is used) +Only one `artifact` file is accepted per request; additional files are ignored. +If the `artifact` part includes a `Content-Length` header, it is validated against the received bytes; it is strongly recommended to include it for uploads. + +The response is JSON: +```json +{"artifact_url":"https://files.kernelci.org///"} +``` + +Filename rules: +- The uploaded filename must match `^[a-z0-9_]+(\.[a-z0-9_]+)*$` (lowercase only). + +The base URL can be overridden with: +``` +KCIDB_STORAGE_URL="https://files-staging.kernelci.org/" +``` + +Example: +```bash +curl -X POST http://localhost:8080/submitartifacts \ + -H "Authorization: Bearer " \ + -F "submission_id=abcd1234" \ + -F "artifact=@./build.log.gz" +``` + +Example responses: +- Success (200): +```json +{"artifact_url":"https://files.kernelci.org//abcd1234/build.log.gz"} +``` +- Missing/invalid fields (400): +```json +{"id":"0","status":"error","message":"submission_id must match [a-z0-9_]+"} +``` +- Unauthorized (401): +```json +{"id":"0","status":"error","message":"JWT is required"} +``` +- Artifact already exists (409): +```json +{"id":"0","status":"error","message":"artifact file already exists"} +``` + +## Environment + +- `JWT_SECRET`: JWT verification secret (required for auth unless empty). +- `KCIDB_STORAGE_URL`: Base URL used in `artifact_url` (default: + `https://files.kernelci.org`). diff --git a/kcidb-restd-rs/src/main.rs b/kcidb-restd-rs/src/main.rs index 38dcd66..f94ee2f 100644 --- a/kcidb-restd-rs/src/main.rs +++ b/kcidb-restd-rs/src/main.rs @@ -27,11 +27,12 @@ KCIDB-Rust REST submissions receiver */ use axum::Router; -use axum::extract::State; +use axum::extract::{Multipart, State}; use axum::http::StatusCode; -use axum::http::header::HeaderMap; +use axum::http::header::{HeaderMap, CONTENT_LENGTH}; use axum::response::IntoResponse; use axum::routing::{get, post}; +use axum::body::Bytes; use axum_server::tls_rustls::RustlsConfig; use clap::Parser; use jsonwebtoken::{DecodingKey, Validation, decode}; @@ -289,6 +290,7 @@ async fn main() { let app = Router::new() .route("/", get(handle_root)) .route("/submit", post(receive_submission)) + .route("/submitartifacts", post(receive_artifacts)) .route("/status", get(submission_status)) .route("/metrics", get(submission_metrics)) .route("/health", get(|| async { "OK" })) @@ -315,6 +317,7 @@ async fn main() { let app = Router::new() .route("/", get(handle_root)) .route("/submit", post(receive_submission)) + .route("/submitartifacts", post(receive_artifacts)) .route("/status", get(submission_status)) .route("/metrics", get(submission_metrics)) .route("/health", get(|| async { "OK" })) @@ -387,6 +390,83 @@ fn generate_answer(status: &str, id: &str, message: Option) -> String { serde_json::to_string(&status).unwrap().to_string() } +#[derive(Debug, Serialize, Deserialize)] +struct ArtifactUploadResponse { + artifact_url: String, +} + +fn storage_base_url() -> String { + let base = std::env::var("KCIDB_STORAGE_URL") + .unwrap_or_else(|_| "https://files.kernelci.org".to_string()); + base.trim_end_matches('/').to_string() +} + +fn sanitize_filename(filename: &str) -> Result { + let base = Path::new(filename) + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("") + .trim(); + if base.is_empty() { + return Err("Missing filename".to_string()); + } + if base == "." || base == ".." { + return Err("Invalid filename".to_string()); + } + let mut parts = base.split('.'); + let first = parts.next().unwrap_or(""); + if first.is_empty() { + return Err("Filename must match [a-z0-9_]+(\\.[a-z0-9_]+)*".to_string()); + } + if !first + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') + { + return Err("Filename must match [a-z0-9_]+(\\.[a-z0-9_]+)*".to_string()); + } + for part in parts { + if part.is_empty() { + return Err("Filename must match [a-z0-9_]+(\\.[a-z0-9_]+)*".to_string()); + } + if !part + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') + { + return Err("Filename must match [a-z0-9_]+(\\.[a-z0-9_]+)*".to_string()); + } + } + Ok(base.to_string()) +} + +fn validate_origin(origin: &str) -> Result<(), String> { + if origin.is_empty() { + return Err("Empty origin".to_string()); + } + if !origin + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') + { + return Err("Invalid origin".to_string()); + } + Ok(()) +} + +fn validate_submission_id(id: &str) -> Result<(), String> { + if id.is_empty() { + return Err("Empty submission_id".to_string()); + } + if id.len() > 64 { + return Err("submission_id too long".to_string()); + } + if !id + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') + { + return Err("submission_id must match [a-z0-9_]+".to_string()); + } + Ok(()) +} + #[derive(serde::Deserialize)] struct StatusQuery { @@ -557,6 +637,238 @@ async fn receive_submission( } } +// POST /submitartifacts multipart/form-data +async fn receive_artifacts( + headers: HeaderMap, + State(state): State>, + mut multipart: Multipart, +) -> impl IntoResponse { + let auth_result = verify_auth(headers, state.clone()); + let origin: String; + match auth_result { + Ok(jwt) => { + origin = jwt.origin; + } + Err(e) => { + println!("Error: {}", e); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some(e), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + state.error_counter.fetch_add(1, Ordering::Relaxed); + return (StatusCode::UNAUTHORIZED, err_json); + } + } + + if let Err(e) = validate_origin(&origin) { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some(e), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } + + let mut submission_id: Option = None; + let mut file_bytes: Option = None; + let mut file_name: Option = None; + let mut file_length_header: Option = None; + + while let Some(field) = match multipart.next_field().await { + Ok(field) => field, + Err(e) => { + println!("Error: {}", e); + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("Invalid multipart form data".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } + } { + if let Some(name) = field.name() { + if name == "submission_id" || name == "submissionid" { + if let Ok(text) = field.text().await { + submission_id = Some(text.trim().to_string()); + } + continue; + } + } + + if field.name() == Some("artifact") && field.file_name().is_some() { + if file_bytes.is_none() { + file_name = field.file_name().map(|s| s.to_string()); + if let Some(header_value) = field.headers().get(CONTENT_LENGTH) { + if let Ok(header_str) = header_value.to_str() { + if let Ok(value) = header_str.parse::() { + file_length_header = Some(value); + } + } + } + match field.bytes().await { + Ok(bytes) => file_bytes = Some(bytes), + Err(_) => { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("Failed to read uploaded file".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } + } + } + } + } + + let submission_id = match submission_id { + Some(id) => id, + None => { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("submission_id is required".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } + }; + + if let Err(e) = validate_submission_id(&submission_id) { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some(e), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } + + let file_name = match file_name { + Some(name) => name, + None => { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("artifact file is required".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } + }; + + let sanitized = match sanitize_filename(&file_name) { + Ok(name) => name, + Err(e) => { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some(e), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } + }; + let file_bytes = match file_bytes { + Some(bytes) => bytes, + None => { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("artifact file is required".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } +}; + if let Some(expected) = file_length_header { + if file_bytes.len() as u64 != expected { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("Artifact Content-Length mismatch".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::BAD_REQUEST, err_json); + } +} + + let artifacts_dir = format!("{}/artifacts", state.directory); + if let Err(e) = std::fs::create_dir_all(&artifacts_dir) { + eprintln!("Failed to create artifacts directory: {}", e); + state.system_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("Internal server error".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap_or_default(); + return (StatusCode::INTERNAL_SERVER_ERROR, err_json); + } + + let local_filename = format!("{}_{}_{}", origin, submission_id, sanitized); + let temp_path = format!("{}/{}.temp", artifacts_dir, local_filename); + let final_path = format!("{}/{}", artifacts_dir, local_filename); + + if Path::new(&final_path).exists() || Path::new(&temp_path).exists() { + state.user_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("artifact file already exists".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap(); + return (StatusCode::CONFLICT, err_json); + } + + if let Err(e) = std::fs::write(&temp_path, &file_bytes) { + eprintln!("Failed to write artifact file: {}", e); + state.system_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("Internal server error".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap_or_default(); + return (StatusCode::INTERNAL_SERVER_ERROR, err_json); + } + if let Err(e) = std::fs::rename(&temp_path, &final_path) { + eprintln!("Failed to rename artifact file: {}", e); + state.system_error_counter.fetch_add(1, Ordering::Relaxed); + let err_status = SubmissionStatus { + id: "0".to_string(), + status: "error".to_string(), + message: Some("Internal server error".to_string()), + }; + let err_json = serde_json::to_string(&err_status).unwrap_or_default(); + return (StatusCode::INTERNAL_SERVER_ERROR, err_json); + } + + let artifact_url = format!( + "{}/{}/{}/{}", + storage_base_url(), + origin, + submission_id, + sanitized + ); + let response = ArtifactUploadResponse { artifact_url }; + let jsonstr = serde_json::to_string(&response).unwrap(); + (StatusCode::OK, jsonstr) +} + #[derive(Debug, Serialize, Deserialize)] struct JWT { origin: String,