From 5354878e51b42fbe3daa6d15b749d5bd7688dc77 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 8 Mar 2026 17:12:38 +0000 Subject: [PATCH 01/15] feat: add script module mode with folder model for Bun and Python Co-Authored-By: Claude Opus 4.6 --- ...dacbabed4c5ae28101e3ae2694f96fd055a91.json | 2 +- ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...2cb01d65b1b46693331ac157dfca0dab6824.json} | 12 +- ...80f13343a20e32a9c9513a0fa0ac39f36fce.json} | 7 +- ...706d78a6f24cb0e614d7d81ba1b643805bf06.json | 2 +- .../20260307000000_add_script_modules.up.sql | 1 + backend/tests/agent_workers.rs | 1 + backend/tests/bun_jobs.rs | 26 +++ backend/tests/dependency_map.rs | 1 + backend/tests/list_jobs.rs | 15 ++ backend/tests/nativets_jobs.rs | 1 + backend/tests/nativets_stress.rs | 1 + backend/tests/python_jobs.rs | 5 + backend/tests/relock_skip.rs | 67 ++++-- backend/tests/volume_tests.rs | 1 + backend/tests/worker.rs | 23 +++ backend/windmill-api-client/src/lib.rs | 41 ++-- backend/windmill-api-scripts/src/scripts.rs | 16 +- backend/windmill-api/openapi.yaml | 29 +++ backend/windmill-api/src/jobs.rs | 4 + backend/windmill-api/src/workspaces_export.rs | 3 + backend/windmill-common/src/cache.rs | 40 +++- backend/windmill-common/src/scripts.rs | 2 + backend/windmill-common/src/worker.rs | 2 +- backend/windmill-queue/src/jobs.rs | 3 + backend/windmill-test-utils/src/lib.rs | 3 + backend/windmill-types/src/jobs.rs | 3 +- backend/windmill-types/src/scripts.rs | 79 ++++++- .../windmill-worker/src/python_executor.rs | 41 ++-- backend/windmill-worker/src/worker.rs | 90 +++++++- backend/windmill-worker/src/worker_flow.rs | 1 + cli/src/commands/script/script.ts | 106 +++++++++- cli/src/commands/sync/sync.ts | 149 ++++++++++++- cli/src/utils/metadata.ts | 77 ++++++- cli/src/utils/resource_folders.ts | 34 +++ .../src/lib/components/ScriptBuilder.svelte | 9 +- .../src/lib/components/ScriptEditor.svelte | 195 +++++++++++++++++- 37 files changed, 992 insertions(+), 102 deletions(-) rename backend/.sqlx/{query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json => query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json} (85%) rename backend/.sqlx/{query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json => query-d122a8092bc48bf4111ee1beab7780f13343a20e32a9c9513a0fa0ac39f36fce.json} (86%) create mode 100644 backend/migrations/20260307000000_add_script_modules.up.sql diff --git a/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json b/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json index da0ce607091e2..90f38c05a7ed1 100644 --- a/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json +++ b/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - false + true ] }, "hash": "2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91" diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9fd94..713ccb9dd35d9 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json b/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json similarity index 85% rename from backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json rename to backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json index 8ec067771613d..8dd4e1ea28a69 100644 --- a/backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json +++ b/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option\", envs AS \"envs: Vec\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar, codebase LIKE '%.esm%' as is_esm FROM script WHERE hash = $1 LIMIT 1", + "query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option\", envs AS \"envs: Vec\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar, codebase LIKE '%.esm%' as is_esm, modules AS \"modules: serde_json::Value\" FROM script WHERE hash = $1 LIMIT 1", "describe": { "columns": [ { @@ -73,6 +73,11 @@ "ordinal": 7, "name": "is_esm", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "modules: serde_json::Value", + "type_info": "Jsonb" } ], "parameters": { @@ -88,8 +93,9 @@ true, false, null, - null + null, + true ] }, - "hash": "7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662" + "hash": "b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824" } diff --git a/backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json b/backend/.sqlx/query-d122a8092bc48bf4111ee1beab7780f13343a20e32a9c9513a0fa0ac39f36fce.json similarity index 86% rename from backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json rename to backend/.sqlx/query-d122a8092bc48bf4111ee1beab7780f13343a20e32a9c9513a0fa0ac39f36fce.json index f1ba61b895943..e55f47156c0de 100644 --- a/backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json +++ b/backend/.sqlx/query-d122a8092bc48bf4111ee1beab7780f13343a20e32a9c9513a0fa0ac39f36fce.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38)", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39)", "describe": { "columns": [], "parameters": { @@ -87,10 +87,11 @@ "Varchar", "Int4", "Bool", - "Int8" + "Int8", + "Jsonb" ] }, "nullable": [] }, - "hash": "b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652" + "hash": "d122a8092bc48bf4111ee1beab7780f13343a20e32a9c9513a0fa0ac39f36fce" } diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json index c96961eac49f3..0a06188897e7d 100644 --- a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - false + true ] }, "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" diff --git a/backend/migrations/20260307000000_add_script_modules.up.sql b/backend/migrations/20260307000000_add_script_modules.up.sql new file mode 100644 index 0000000000000..c8bd0ea3bba53 --- /dev/null +++ b/backend/migrations/20260307000000_add_script_modules.up.sql @@ -0,0 +1 @@ +ALTER TABLE script ADD COLUMN IF NOT EXISTS modules JSONB; diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index 4f12d5f42c6a2..f422ed3a1295c 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -21,6 +21,7 @@ fn bun_code(code: &str) -> RawCode { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, } } diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 1617babb8139a..31cd767cf00a7 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -34,6 +34,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -70,6 +71,7 @@ export function main(name: string, count: number) { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = RunJob::from(job) @@ -112,6 +114,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -144,6 +147,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -177,6 +181,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -217,6 +222,7 @@ export async function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -256,6 +262,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -288,6 +295,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -330,6 +338,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let completed = run_job_in_new_worker_until_complete(&db, false, job, port).await; @@ -370,6 +379,7 @@ export function notMain() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let completed = run_job_in_new_worker_until_complete(&db, false, job, port).await; @@ -410,6 +420,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let completed = run_job_in_new_worker_until_complete(&db, false, job, port).await; @@ -449,6 +460,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -486,6 +498,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -528,6 +541,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -626,6 +640,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -661,6 +676,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -707,6 +723,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -755,6 +772,7 @@ export function main(x: number) { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); // x=5, main adds 10 = 15 @@ -805,6 +823,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -850,6 +869,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -1175,6 +1195,7 @@ export function main(name: string) { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = RunJob::from(job) @@ -1238,6 +1259,7 @@ export function main(name: string) { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = RunJob::from(job) @@ -1513,6 +1535,7 @@ module.exports.main = function() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = RunJob::from(job) @@ -1554,6 +1577,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = RunJob::from(job) @@ -1604,6 +1628,7 @@ module.exports.main = function() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); use std::sync::atomic::Ordering; @@ -1664,6 +1689,7 @@ export function main() { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); use std::sync::atomic::Ordering; diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index f3c03aa93226b..f1ddf78d00d92 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -46,6 +46,7 @@ mod dependency_map { has_preprocessor: None, on_behalf_of_email: None, assets: vec![], + modules: None, } } async fn init(db: Pool) -> (windmill_api_client::Client, u16, ApiServer) { diff --git a/backend/tests/list_jobs.rs b/backend/tests/list_jobs.rs index 65451bcbb7b12..5db9e3cb14666 100644 --- a/backend/tests/list_jobs.rs +++ b/backend/tests/list_jobs.rs @@ -67,6 +67,7 @@ async fn test_list_jobs_without_include_args(db: Pool) -> anyhow::Resu concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(42)) .push(&db) @@ -123,6 +124,7 @@ async fn test_list_jobs_with_include_args(db: Pool) -> anyhow::Result< concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(42)) .push(&db) @@ -193,6 +195,7 @@ async fn test_list_jobs_completed_with_include_args(db: Pool) -> anyho concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(42)) .run_until_complete(&db, false, port) @@ -265,6 +268,7 @@ async fn test_list_jobs_mixed_queue_and_completed(db: Pool) -> anyhow: concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("completed_arg", json!("completed_value")) .run_until_complete(&db, false, port) @@ -285,6 +289,7 @@ async fn test_list_jobs_mixed_queue_and_completed(db: Pool) -> anyhow: concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("queued_arg", json!("queued_value")) .push(&db) @@ -367,6 +372,7 @@ async fn test_list_jobs_multiple_queued_with_include_args(db: Pool) -> concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(1)) .push(&db) @@ -384,6 +390,7 @@ async fn test_list_jobs_multiple_queued_with_include_args(db: Pool) -> concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("y", json!(2)) .push(&db) @@ -457,6 +464,7 @@ async fn test_queue_list_without_include_args(db: Pool) -> anyhow::Res concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(42)) .push(&db) @@ -519,6 +527,7 @@ async fn test_queue_list_with_include_args(db: Pool) -> anyhow::Result concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(42)) .push(&db) @@ -588,6 +597,7 @@ async fn test_queue_list_multiple_jobs_with_include_args(db: Pool) -> concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!("value_a")) .push(&db) @@ -605,6 +615,7 @@ async fn test_queue_list_multiple_jobs_with_include_args(db: Pool) -> concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("b", json!("value_b")) .push(&db) @@ -686,6 +697,7 @@ async fn test_completed_list_without_include_args(db: Pool) -> anyhow: concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(42)) .run_until_complete(&db, false, port) @@ -751,6 +763,7 @@ async fn test_completed_list_with_include_args(db: Pool) -> anyhow::Re concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("x", json!(42)) .run_until_complete(&db, false, port) @@ -825,6 +838,7 @@ async fn test_completed_list_multiple_jobs_with_include_args( concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!("completed_a")) .run_until_complete(&db, false, port) @@ -845,6 +859,7 @@ async fn test_completed_list_multiple_jobs_with_include_args( concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("b", json!("completed_b")) .run_until_complete(&db, false, port) diff --git a/backend/tests/nativets_jobs.rs b/backend/tests/nativets_jobs.rs index eaf3a96652f5a..5c57e5c0c844b 100644 --- a/backend/tests/nativets_jobs.rs +++ b/backend/tests/nativets_jobs.rs @@ -39,6 +39,7 @@ fn nativets_code(content: &str) -> JobPayload { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }) } diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 082a43be2cdd3..a175d97af4a75 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -156,6 +156,7 @@ async fn push_job(db: &Pool, content: &str, args: &serde_json::Value) cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let tx = PushIsolationLevel::IsolatedRoot(db.clone()); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 22c2313b91223..34302450ff449 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -194,6 +194,7 @@ def main(): cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -245,6 +246,7 @@ def main(): cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -281,6 +283,7 @@ def main(): cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -321,6 +324,7 @@ def main(): cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) @@ -359,6 +363,7 @@ def main(): cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port) diff --git a/backend/tests/relock_skip.rs b/backend/tests/relock_skip.rs index ce274796bc6e1..aed3d79fefaf0 100644 --- a/backend/tests/relock_skip.rs +++ b/backend/tests/relock_skip.rs @@ -44,6 +44,7 @@ mod relock_skip { has_preprocessor: None, on_behalf_of_email: None, assets: vec![], + modules: None, } } @@ -57,13 +58,10 @@ mod relock_skip { pattern: &str, after: chrono::DateTime, ) -> i64 { - let logs = sqlx::query_scalar!( - "SELECT logs FROM job_logs WHERE created_at > $1", - after - ) - .fetch_all(db) - .await - .unwrap(); + let logs = sqlx::query_scalar!("SELECT logs FROM job_logs WHERE created_at > $1", after) + .fetch_all(db) + .await + .unwrap(); logs.iter() .filter_map(|l| l.as_ref()) @@ -270,8 +268,14 @@ def main(): // We allow up to 3 skips from cascade re-triggers. let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; let relocking_count = count_pattern_in_job_logs(&db, "Relocking", before).await; - assert!(skipping_count <= 3, "First deployment should have at most 3 skips from cascade"); - assert!(relocking_count >= 3, "First deployment should have at least 3 relocking jobs"); + assert!( + skipping_count <= 3, + "First deployment should have at most 3 skips from cascade" + ); + assert!( + relocking_count >= 3, + "First deployment should have at least 3 relocking jobs" + ); // Step 2: Redeploy default workspace deps again - should SKIP let before = chrono::Utc::now(); @@ -289,7 +293,10 @@ def main(): in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; - assert!(skipping_count >= 3, "Second deployment of same content should skip at least 3 times"); + assert!( + skipping_count >= 3, + "Second deployment of same content should skip at least 3 times" + ); // Step 3: Redeploy default workspace deps with different content - should NOT skip let before = chrono::Utc::now(); @@ -308,8 +315,14 @@ def main(): let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; let relocking_count = count_pattern_in_job_logs(&db, "Relocking", before).await; - assert!(skipping_count <= 4, "Changed content should have at most 3 skips from cascade"); - assert!(relocking_count >= 3, "Changed content should trigger at least 3 relocking jobs"); + assert!( + skipping_count <= 4, + "Changed content should have at most 3 skips from cascade" + ); + assert!( + relocking_count >= 3, + "Changed content should trigger at least 3 relocking jobs" + ); // Step 4: Deploy named workspace deps first time - should relock (no hash exists yet) // Named deps trigger exactly 3 independent objects with no cascade @@ -329,8 +342,14 @@ def main(): let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; let relocking_count = count_pattern_in_job_logs(&db, "Relocking", before).await; - assert_eq!(skipping_count, 0, "Named workspace deps first deployment should not skip"); - assert!(relocking_count > 0, "Named workspace deps first deployment should relock"); + assert_eq!( + skipping_count, 0, + "Named workspace deps first deployment should not skip" + ); + assert!( + relocking_count > 0, + "Named workspace deps first deployment should relock" + ); // Step 5: Deploy named workspace deps again with no change - should SKIP let before = chrono::Utc::now(); @@ -350,8 +369,14 @@ def main(): let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; let relocking_count = count_pattern_in_job_logs(&db, "Relocking", before).await; - assert!(skipping_count > 0, "Named workspace deps second deployment should skip"); - assert_eq!(relocking_count, 0, "Named workspace deps second deployment should not relock"); + assert!( + skipping_count > 0, + "Named workspace deps second deployment should skip" + ); + assert_eq!( + relocking_count, 0, + "Named workspace deps second deployment should not relock" + ); // Step 6: Deploy named workspace deps with small change - should NOT skip let before = chrono::Utc::now(); @@ -370,8 +395,14 @@ def main(): let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; let relocking_count = count_pattern_in_job_logs(&db, "Relocking", before).await; - assert_eq!(skipping_count, 0, "Named workspace deps with change should not skip"); - assert!(relocking_count > 0, "Named workspace deps with change should relock"); + assert_eq!( + skipping_count, 0, + "Named workspace deps with change should not skip" + ); + assert!( + relocking_count > 0, + "Named workspace deps with change should relock" + ); Ok(()) } diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs index 9df30c561511d..27f82dfb386cb 100644 --- a/backend/tests/volume_tests.rs +++ b/backend/tests/volume_tests.rs @@ -596,6 +596,7 @@ export function main() { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, }); let result = run_job_in_new_worker_until_complete(&db, false, job, port).await; diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 9548986a5a69d..99e4498c02ca0 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -867,6 +867,7 @@ func main(derp string) (string, error) { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("derp", json!("world")) .run_until_complete(&db, false, port) @@ -905,6 +906,7 @@ fn main(world: String) -> Result { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, + modules: None, })) .arg("world", json!("Hyrule")) .run_until_complete(&db, false, port) @@ -949,6 +951,7 @@ class Script concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("world", json!("Arakis")) .arg("b", json!(3)) @@ -985,6 +988,7 @@ echo "hello $msg" concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("msg", json!("world")) .run_until_complete(&db, false, port) @@ -1022,6 +1026,7 @@ echo "$result" concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .run_until_complete(&db, false, port) .await; @@ -1056,6 +1061,7 @@ echo "$result" concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .run_until_complete(&db, false, port) .await; @@ -1093,6 +1099,7 @@ def main [ msg: string ] { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("msg", json!("world")) .run_until_complete(&db, false, port) @@ -1147,6 +1154,7 @@ def main [ concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!("3")) .arg("b", json!("null")) @@ -1210,6 +1218,7 @@ public class Main { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!(3)) .arg("b", json!(3.0)) @@ -1247,6 +1256,7 @@ export async function main(name: string): Promise { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("name", json!("world")) .run_until_complete(&db, false, port) @@ -1284,6 +1294,7 @@ export async function main(a: number, b: number): Promise { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!(3)) .arg("b", json!(7)) @@ -1322,6 +1333,7 @@ export async function main(items: string[]): Promise<{ count: number; items: str concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("items", json!(["a", "b", "c"])) .run_until_complete(&db, false, port) @@ -1360,6 +1372,7 @@ export async function main(a: Date) { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .run_until_complete(&db, false, port) @@ -1395,6 +1408,7 @@ SELECT 'hello ' || $1::text AS result; concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("name", json!("world")) .arg( @@ -1435,6 +1449,7 @@ SELECT ? AS result; concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("name", json!("world")) .arg( @@ -1475,6 +1490,7 @@ export async function main(name: string): Promise { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("name", json!("world")) .run_until_complete(&db, false, port) @@ -1510,6 +1526,7 @@ Write-Output "hello $msg" concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("msg", json!("world")) .run_until_complete(&db, false, port) @@ -1546,6 +1563,7 @@ function main(string $name): string { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("name", json!("world")) .run_until_complete(&db, false, port) @@ -1583,6 +1601,7 @@ end concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("name", json!("world")) .run_until_complete(&db, false, port) @@ -1619,6 +1638,7 @@ export async function main(a: Date) { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .run_until_complete(&db, false, port) @@ -1655,6 +1675,7 @@ export async function main(a: Date) { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .run_until_complete(&db, false, port) @@ -1707,6 +1728,7 @@ export function main(name: string) { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, + modules: None, })) .arg("name", json!("World")) .run_until_complete(&db, false, port) @@ -1752,6 +1774,7 @@ def main(a: datetime, b: bytes): concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .arg("b", json!("dGVzdA==")) diff --git a/backend/windmill-api-client/src/lib.rs b/backend/windmill-api-client/src/lib.rs index 4e37958b7eb7c..3a84d14d1c486 100644 --- a/backend/windmill-api-client/src/lib.rs +++ b/backend/windmill-api-client/src/lib.rs @@ -17,10 +17,7 @@ pub struct Client { impl Client { /// Create a new client with an existing reqwest::Client pub fn new_with_client(baseurl: &str, client: reqwest::Client) -> Self { - Self { - baseurl: baseurl.to_string(), - client, - } + Self { baseurl: baseurl.to_string(), client } } /// Get the base URL @@ -49,7 +46,10 @@ impl Client { if response.status().is_success() { Ok(response.text().await?) } else { - Err(Error::UnexpectedResponse(response.status().as_u16(), response.text().await.unwrap_or_default())) + Err(Error::UnexpectedResponse( + response.status().as_u16(), + response.text().await.unwrap_or_default(), + )) } } @@ -69,7 +69,10 @@ impl Client { if response.status().is_success() { Ok(response.text().await?) } else { - Err(Error::UnexpectedResponse(response.status().as_u16(), response.text().await.unwrap_or_default())) + Err(Error::UnexpectedResponse( + response.status().as_u16(), + response.text().await.unwrap_or_default(), + )) } } @@ -97,7 +100,10 @@ impl Client { if response.status().is_success() { Ok(response.json().await?) } else { - Err(Error::UnexpectedResponse(response.status().as_u16(), response.text().await.unwrap_or_default())) + Err(Error::UnexpectedResponse( + response.status().as_u16(), + response.text().await.unwrap_or_default(), + )) } } @@ -117,7 +123,10 @@ impl Client { if response.status().is_success() { Ok(response.text().await?) } else { - Err(Error::UnexpectedResponse(response.status().as_u16(), response.text().await.unwrap_or_default())) + Err(Error::UnexpectedResponse( + response.status().as_u16(), + response.text().await.unwrap_or_default(), + )) } } @@ -139,7 +148,10 @@ impl Client { if response.status().is_success() { Ok(response.text().await?) } else { - Err(Error::UnexpectedResponse(response.status().as_u16(), response.text().await.unwrap_or_default())) + Err(Error::UnexpectedResponse( + response.status().as_u16(), + response.text().await.unwrap_or_default(), + )) } } @@ -151,7 +163,10 @@ impl Client { if response.status().is_success() { Ok(response.json().await?) } else { - Err(Error::UnexpectedResponse(response.status().as_u16(), response.text().await.unwrap_or_default())) + Err(Error::UnexpectedResponse( + response.status().as_u16(), + response.text().await.unwrap_or_default(), + )) } } } @@ -388,6 +403,8 @@ pub mod types { pub visible_to_runner_only: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub ws_error_handler_muted: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modules: Option>, } /// Script arguments (used in schedules) @@ -555,12 +572,12 @@ pub mod types { Static { #[serde(rename = "type")] type_: String, - value: serde_json::Value + value: serde_json::Value, }, Javascript { #[serde(rename = "type")] type_: String, - expr: String + expr: String, }, } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index d93b33670dec2..2b27fd393d2bf 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -65,7 +65,8 @@ use windmill_common::{ schema::should_validate_schema, scripts::{ to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash, - ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred, + ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptModule, + ScriptWithStarred, }, users::username_to_permissioned_as, utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath}, @@ -124,6 +125,9 @@ pub struct ScriptWDraft { #[serde(skip_serializing_if = "Option::is_none")] #[sqlx(json(nullable))] pub assets: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub modules: Option>, #[serde(flatten)] #[sqlx(flatten)] pub runnable_settings: SR, @@ -178,6 +182,7 @@ impl ScriptWDraft { has_preprocessor: self.has_preprocessor, on_behalf_of_email: self.on_behalf_of_email, assets: self.assets, + modules: self.modules, }) } } @@ -894,8 +899,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39)", &w_id, &hash.0, ns.path, @@ -937,7 +942,8 @@ async fn create_script_internal<'c>( guarded_debounce_key, guarded_debounce_delay_s, ns.cache_ignore_s3_path, - runnable_settings_handle + runnable_settings_handle, + ns.modules.as_ref().and_then(|m| serde_json::to_value(m).ok()) ) .execute(&mut *tx) .await?; @@ -1387,7 +1393,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets, debounce_key, debounce_delay_s FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets, modules, debounce_key, debounce_delay_s FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 17502767788fc..607ba397eebe7 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -18681,6 +18681,12 @@ components: type: boolean on_behalf_of_email: type: string + modules: + type: object + nullable: true + description: "Additional script modules keyed by relative file path" + additionalProperties: + $ref: "#/components/schemas/ScriptModule" required: - hash @@ -18799,6 +18805,12 @@ components: alt_access_type: type: string enum: [r, w, rw] + modules: + type: object + nullable: true + description: "Additional script modules keyed by relative file path" + additionalProperties: + $ref: "#/components/schemas/ScriptModule" required: - path @@ -19993,6 +20005,23 @@ components: # for related places search: ADD_NEW_LANG ] + ScriptModule: + type: object + description: "An additional module file associated with a script" + properties: + content: + type: string + description: "The source code content of this module" + language: + $ref: "#/components/schemas/ScriptLang" + lock: + type: string + nullable: true + description: "Lock file content for this module's dependencies" + required: + - content + - language + Preview: type: object properties: diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index ac5a9a306bef8..6a72c383630f3 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -3591,6 +3591,7 @@ pub async fn run_workflow_as_code( dedicated_worker: None, // TODO(debouncing): enable for this mode debouncing_settings: DebouncingSettings::default(), + modules: None, }), Some(job.tag.clone()), None, @@ -4626,6 +4627,7 @@ async fn run_preview_script( cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: preview.dedicated_worker, + modules: None, }), }, push_args, @@ -4967,6 +4969,7 @@ async fn run_bundle_preview_script( dedicated_worker: preview.dedicated_worker, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), + modules: None, }), PushArgs::from(&args), authed.display_username(), @@ -5760,6 +5763,7 @@ async fn run_dynamic_select( dedicated_worker: None, concurrency_settings: ConcurrencySettings::default().into(), debouncing_settings: DebouncingSettings::default(), + modules: None, }), PushArgs::from(&request.args.unwrap_or_default()), authed.display_username(), diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 091c465a6131e..5a067620f9734 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -97,6 +97,8 @@ struct ScriptMetadata { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub modules: Option>, #[serde(flatten)] pub concurrency_settings: ConcurrencySettings, #[serde(flatten)] @@ -487,6 +489,7 @@ pub(crate) async fn tarball_workspace( codebase: script.codebase, has_preprocessor: script.has_preprocessor, on_behalf_of_email: script.on_behalf_of_email, + modules: script.modules, }; let metadata_str = serde_json::to_string_pretty(&metadata).unwrap(); archive diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 27af55ea01416..89010558893fa 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -12,7 +12,7 @@ use crate::{ error, flows::{FlowNodeId, FlowValue}, schema::SchemaValidator, - scripts::{ScriptHash, ScriptLang}, + scripts::{ScriptHash, ScriptLang, ScriptModule}, }; use anyhow::anyhow; use serde_json::value::to_raw_value; @@ -335,6 +335,7 @@ impl FlowData { pub struct ScriptData { pub lock: Option, pub code: String, + pub modules: Option>, } #[derive(Debug, Clone)] @@ -357,6 +358,7 @@ pub struct RawScript { pub content: String, pub lock: Option, pub meta: Option, + pub modules: Option>, } #[derive(Debug, Deserialize, Serialize)] @@ -364,17 +366,28 @@ pub struct RawScriptApi { pub content: String, pub lock: Option, pub meta: Option, + pub modules: Option>, } impl From for RawScriptApi { fn from(value: RawScript) -> Self { - RawScriptApi { content: value.content, lock: value.lock, meta: value.meta } + RawScriptApi { + content: value.content, + lock: value.lock, + meta: value.meta, + modules: value.modules, + } } } impl From for RawScript { fn from(value: RawScriptApi) -> Self { - RawScript { content: value.content, lock: value.lock, meta: value.meta } + RawScript { + content: value.content, + lock: value.lock, + meta: value.meta, + modules: value.modules, + } } } @@ -632,7 +645,8 @@ pub mod script { schema AS \"schema: String\", \ schema_validation AS \"schema_validation: bool\", \ codebase LIKE '%.tar' as use_tar, \ - codebase LIKE '%.esm%' as is_esm \ + codebase LIKE '%.esm%' as is_esm, \ + modules AS \"modules: serde_json::Value\" \ FROM script WHERE hash = $1 LIMIT 1", hash.0 ) @@ -644,6 +658,7 @@ pub mod script { Ok(RawScript { content: r.content, lock: r.lock, + modules: r.modules.and_then(|v| serde_json::from_value(v).ok()), meta: Some(ScriptMetadata { language: r.language, envs: r.envs, @@ -822,6 +837,7 @@ pub mod job { _ => Ok(RawData::Script(Arc::new(ScriptData { code: code.unwrap_or_default(), lock, + modules: None, }))), }) }; @@ -999,7 +1015,7 @@ const _: () = { let content = src.get_utf8("code.txt")?; let lock = src.get_utf8("lock.txt").ok(); let meta = src.get_json("info.json").ok(); - Ok(Self { content, lock, meta }) + Ok(Self { content, lock, meta, modules: None }) } } @@ -1007,7 +1023,7 @@ const _: () = { type Untrusted = RawScript; fn resolve(src: Self::Untrusted) -> error::Result { - Ok(ScriptData { code: src.content, lock: src.lock }) + Ok(ScriptData { code: src.content, lock: src.lock, modules: src.modules }) } fn export(&self, dst: &impl Storage) -> error::Result<()> { @@ -1033,7 +1049,11 @@ const _: () = { return Err(error::Error::internal_err("Invalid script src".to_string())); }; Ok(ScriptFull { - data: Arc::new(ScriptData { code: src.content, lock: src.lock }), + data: Arc::new(ScriptData { + code: src.content, + lock: src.lock, + modules: src.modules, + }), meta: Arc::new(meta), }) } @@ -1063,7 +1083,11 @@ const _: () = { FlowData::from_raw(flow).map(Arc::new).map(Self::Flow) } RawNode { raw_code: Some(code), raw_lock: lock, .. } => { - Ok(Self::Script(Arc::new(ScriptData { code, lock }))) + Ok(Self::Script(Arc::new(ScriptData { + code, + lock, + modules: None, + }))) } _ => Err(error::Error::internal_err( "Invalid raw data src".to_string(), diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index cfff28dbcbc17..ce54abdfc61d1 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -106,6 +106,7 @@ pub async fn prefetch_cached_script( has_preprocessor: script.has_preprocessor, on_behalf_of_email: script.on_behalf_of_email, assets: script.assets, + modules: script.modules, runnable_settings: ScriptRunnableSettingsInline { concurrency_settings: concurrency_settings.maybe_fallback( script.runnable_settings.concurrency_key, @@ -418,6 +419,7 @@ pub async fn clone_script<'c>( on_behalf_of_email: s.on_behalf_of_email, preserve_on_behalf_of: None, assets: s.assets, + modules: s.modules, }; let new_hash = hash_script(&ns); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index b3876fbcfa92f..8b20d331c0bef 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1361,7 +1361,7 @@ pub async fn fetch_raw_script_from_app_query( .await .map_err(Into::into) .and_then(unwrap_or_error(&loc, "Application script", id)) - .map(|r| RawScript { content: r.code, lock: r.lock, meta: None }) + .map(|r| RawScript { content: r.code, lock: r.lock, meta: None, modules: None }) } pub async fn insert_ping_query( diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 806e3e99a25bf..f94c7eb0ee2e7 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -466,6 +466,7 @@ pub async fn push_init_job<'c>( dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), + modules: None, }), PushArgs::from(&ehm), worker_name, @@ -523,6 +524,7 @@ pub async fn push_periodic_bash_job<'c>( dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), + modules: None, }), PushArgs::from(&ehm), worker_name, @@ -4763,6 +4765,7 @@ async fn push_inner<'c, 'd>( dedicated_worker, concurrency_settings, debouncing_settings, + modules, }) => JobPayloadUntagged { runnable_id: hash, runnable_path: path, diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 5d5c80b47029c..fb96ae0a097f9 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -628,6 +628,7 @@ pub async fn assert_lockfile( has_preprocessor: None, on_behalf_of_email: None, assets: vec![], + modules: None, }, ) .await @@ -725,6 +726,7 @@ pub async fn run_deployed_relative_imports( has_preprocessor: None, on_behalf_of_email: None, assets: vec![], + modules: None, }, ) .await @@ -810,6 +812,7 @@ pub async fn run_preview_relative_imports( windmill_common::runnable_settings::ConcurrencySettings::default().into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, })) .push(&db2) .await; diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index ffd3cc6ef39d0..28c32426957f9 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -10,7 +10,7 @@ use crate::{ flow_status::{FlowStatus, RestartedFrom}, flows::{FlowNodeId, FlowValue, Retry}, runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, - scripts::{ScriptHash, ScriptLang}, + scripts::{ScriptHash, ScriptLang, ScriptModule}, }; #[derive(Debug, Deserialize, Clone)] @@ -469,6 +469,7 @@ pub struct RawCode { pub concurrency_settings: ConcurrencySettingsWithCustom, #[serde(flatten)] pub debouncing_settings: DebouncingSettings, + pub modules: Option>, } impl JobPayload { diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index bcbe0ede33f4c..a7d37bd727fb8 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -9,11 +9,21 @@ use itertools::Itertools; use serde::de::Error as _; use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; +use std::collections::HashMap; + use crate::{ assets::AssetWithAltAccessType, runnable_settings::{ConcurrencySettings, DebouncingSettings}, }; +#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +pub struct ScriptModule { + pub content: String, + pub language: ScriptLang, + #[serde(skip_serializing_if = "Option::is_none")] + pub lock: Option, +} + #[derive( Serialize, Deserialize, @@ -105,15 +115,15 @@ impl ScriptLang { pub fn is_native(&self) -> bool { matches!( self, - ScriptLang::Bunnative | - ScriptLang::Nativets | - ScriptLang::Postgresql | - ScriptLang::Mysql | - ScriptLang::Graphql | - ScriptLang::Snowflake | - ScriptLang::Mssql | - ScriptLang::Bigquery | - ScriptLang::OracleDB + ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Postgresql + | ScriptLang::Mysql + | ScriptLang::Graphql + | ScriptLang::Snowflake + | ScriptLang::Mssql + | ScriptLang::Bigquery + | ScriptLang::OracleDB ) } @@ -360,6 +370,9 @@ pub struct Script { #[serde(skip_serializing_if = "Option::is_none")] #[sqlx(json(nullable))] pub assets: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub modules: Option>, #[serde(flatten)] #[sqlx(flatten)] pub runnable_settings: SR, @@ -454,7 +467,7 @@ impl Hash for Schema { } } -#[derive(Serialize, Deserialize, Hash, Debug)] +#[derive(Serialize, Deserialize, Debug)] pub struct NewScript { pub path: String, pub parent_hash: Option, @@ -493,6 +506,52 @@ pub struct NewScript { pub preserve_on_behalf_of: Option, #[serde(skip_serializing_if = "Option::is_none")] pub assets: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub modules: Option>, +} + +impl Hash for NewScript { + fn hash(&self, state: &mut H) { + self.path.hash(state); + self.parent_hash.hash(state); + self.summary.hash(state); + self.description.hash(state); + self.content.hash(state); + self.schema.hash(state); + self.is_template.hash(state); + self.lock.hash(state); + self.language.hash(state); + self.kind.hash(state); + self.tag.hash(state); + self.draft_only.hash(state); + self.envs.hash(state); + self.concurrency_settings.hash(state); + self.debouncing_settings.hash(state); + self.cache_ttl.hash(state); + self.cache_ignore_s3_path.hash(state); + self.dedicated_worker.hash(state); + self.ws_error_handler_muted.hash(state); + self.priority.hash(state); + self.timeout.hash(state); + self.delete_after_use.hash(state); + self.restart_unless_cancelled.hash(state); + self.deployment_message.hash(state); + self.visible_to_runner_only.hash(state); + self.no_main_func.hash(state); + self.codebase.hash(state); + self.has_preprocessor.hash(state); + self.on_behalf_of_email.hash(state); + self.preserve_on_behalf_of.hash(state); + self.assets.hash(state); + if let Some(modules) = &self.modules { + let mut sorted: Vec<_> = modules.iter().collect(); + sorted.sort_by_key(|(k, _)| k.clone()); + for (k, v) in sorted { + k.hash(state); + v.hash(state); + } + } + } } fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 619087659d1cd..3e338e3d2d377 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -542,6 +542,32 @@ async fn postinstall( Ok(()) } +/// Compute the directory (relative to job_dir) where Python writes the main script. +/// Module files must be placed in this same directory for relative imports to work. +pub fn compute_python_module_dir(script_path: &str) -> String { + let script_path_splitted = script_path.split("/").map(|x| { + if x.starts_with(|x: char| x.is_ascii_digit()) { + format!("_{}", x) + } else { + x.to_string() + } + }); + let dirs_full = script_path_splitted + .clone() + .take(script_path_splitted.clone().count() - 1) + .join("/") + .replace("-", "_") + .replace("@", "."); + if dirs_full.len() > 0 { + dirs_full + .strip_prefix("/") + .unwrap_or(&dirs_full) + .to_string() + } else { + "tmp".to_string() + } +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_python_job( requirements_o: Option<&String>, @@ -958,6 +984,7 @@ async fn prepare_wrapper( let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content); + let dirs = compute_python_module_dir(script_path); let script_path_splitted = script_path.split("/").map(|x| { if x.starts_with(|x: char| x.is_ascii_digit()) { format!("_{}", x) @@ -965,20 +992,6 @@ async fn prepare_wrapper( x.to_string() } }); - let dirs_full = script_path_splitted - .clone() - .take(script_path_splitted.clone().count() - 1) - .join("/") - .replace("-", "_") - .replace("@", "."); - let dirs = if dirs_full.len() > 0 { - dirs_full - .strip_prefix("/") - .unwrap_or(&dirs_full) - .to_string() - } else { - "tmp".to_string() - }; let last = script_path_splitted .clone() .last() diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 42020642011e4..cc584ff33d200 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -24,6 +24,7 @@ use windmill_common::runtime_assets::init_runtime_asset_loop; use windmill_common::runtime_assets::register_runtime_asset; use windmill_common::scripts::hash_to_codebase_id; use windmill_common::scripts::is_special_codebase_hash; +use windmill_common::scripts::ScriptModule; use windmill_common::utils::report_critical_error; use windmill_common::utils::retrieve_common_worker_prefix; use windmill_common::worker::error_to_value; @@ -3538,6 +3539,7 @@ pub struct ContentReqLangEnvs { pub envs: Option>, pub codebase: Option, pub schema: Option, + pub modules: Option>, } pub async fn get_hub_script_content_and_requirements( @@ -3557,6 +3559,7 @@ pub async fn get_hub_script_content_and_requirements( envs: None, codebase: None, schema: Some(script.schema.get().to_string()), + modules: None, }) } @@ -3577,6 +3580,7 @@ pub async fn get_script_content_by_hash( Some(_) => Some(script_hash.to_string()), }, schema: None, + modules: data.modules.clone(), }) } @@ -3706,7 +3710,7 @@ async fn handle_code_execution_job( // Box::pin the script fetching match to prevent large enum on stack let ( - ScriptData { code, lock }, + ScriptData { code, lock, modules }, ScriptMetadata { language, envs, codebase, schema_validator, schema }, ) = match job.kind { JobKind::Preview => { @@ -3731,14 +3735,14 @@ async fn handle_code_execution_job( } } JobKind::Script_Hub => { - let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = + let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema, .. } = Box::pin(get_hub_script_content_and_requirements( job.runnable_path.as_ref(), conn.as_sql(), )) .await?; - data = ScriptData { code: content, lock: lockfile }; + data = ScriptData { code: content, lock: lockfile, modules: None }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; (&data, &metadata) } @@ -3783,13 +3787,20 @@ async fn handle_code_execution_job( .as_ref() .ok_or_else(|| Error::internal_err("expected script path".to_string()))?; if script_path.starts_with("hub/") { - let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - Box::pin(get_hub_script_content_and_requirements( - Some(script_path), - conn.as_sql(), - )) - .await?; - data = ScriptData { code: content, lock: lockfile }; + let ContentReqLangEnvs { + content, + lockfile, + language, + envs, + codebase, + schema, + .. + } = Box::pin(get_hub_script_content_and_requirements( + Some(script_path), + conn.as_sql(), + )) + .await?; + data = ScriptData { code: content, lock: lockfile, modules: None }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; (&data, &metadata) @@ -3853,11 +3864,57 @@ async fn handle_code_execution_job( envs, codebase, lock, + modules, false, ) .await } +/// Compute the directory (relative to job_dir) where Python writes the main script. +/// Module files must be placed in this same directory for relative imports to work. +fn compute_python_module_dir(script_path: &str) -> String { + let parts: Vec = script_path + .split("/") + .map(|x| { + if x.starts_with(|x: char| x.is_ascii_digit()) { + format!("_{}", x) + } else { + x.to_string() + } + }) + .collect(); + let dirs_full = parts[..parts.len().saturating_sub(1)] + .join("/") + .replace("-", "_") + .replace("@", "."); + if dirs_full.len() > 0 { + dirs_full + .strip_prefix("/") + .unwrap_or(&dirs_full) + .to_string() + } else { + "tmp".to_string() + } +} + +pub async fn write_module_files( + job_dir: &str, + modules: &std::collections::HashMap, + base_dir: Option<&str>, +) -> error::Result<()> { + for (relpath, module) in modules { + let full_path = match base_dir { + Some(dir) => format!("{}/{}/{}", job_dir, dir, relpath), + None => format!("{}/{}", job_dir, relpath), + }; + if let Some(parent) = std::path::Path::new(&full_path).parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(&full_path, &module.content).await?; + } + Ok(()) +} + pub async fn run_language_executor( job: &MiniPulledJob, conn: &Connection, @@ -3880,8 +3937,19 @@ pub async fn run_language_executor( envs: &Option>, codebase: &Option, lock: &Option, + modules: &Option>, run_inline: bool, ) -> error::Result> { + if let Some(modules) = modules { + let base_dir = if language == Some(ScriptLang::Python3) { + let script_path = crate::common::use_flow_root_path(job.runnable_path()); + Some(compute_python_module_dir(&script_path)) + } else { + None + }; + write_module_files(job_dir, modules, base_dir.as_deref()).await?; + } + if language == Some(ScriptLang::Postgresql) { return do_postgresql( job, @@ -4920,6 +4988,7 @@ pub fn init_worker_internal_server_inline_utils( &None, &None, &None, + &None, true, ) .await @@ -5000,6 +5069,7 @@ pub fn init_worker_internal_server_inline_utils( &content_info.envs, &content_info.codebase, &content_info.lockfile, + &content_info.modules, true, ) .await diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 33dafbb4820ff..c517d06f2b4f0 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -4997,6 +4997,7 @@ pub fn raw_script_to_payload( concurrency_settings, // TODO: Should this have debouncing? debouncing_settings: DebouncingSettings::default(), + modules: None, }), tag, delete_after_use, diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 9c6f094b41788..08b2497085d36 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -9,6 +9,7 @@ import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; +import * as path from "node:path"; import { stringify as yamlStringify } from "yaml"; import { deepEqual } from "../../utils/utils.ts"; import * as wmill from "../../../gen/services.gen.ts"; @@ -51,13 +52,16 @@ import fs from "node:fs"; import { createTarBlob, type TarEntry } from "../../utils/tar.ts"; import { execSync } from "node:child_process"; -import { NewScript, Script } from "../../../gen/types.gen.ts"; +import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts"; import { isRawAppBackendPath as isRawAppBackendPathInternal, isAppInlineScriptPath as isAppInlineScriptPathInternal, isFlowInlineScriptPath as isFlowInlineScriptPathInternal, isFlowPath, isAppPath, + isScriptModulePath, + buildModuleFolderPath, + getModuleFolderSuffix, } from "../../utils/resource_folders.ts"; export interface ScriptFile { @@ -229,6 +233,7 @@ export async function handleFile( !isAppInlineScriptPath(path) && !isFlowInlineScriptPath(path) && !isRawAppBackendPath(path) && + !isScriptModulePath(path) && exts.some((exts) => path.endsWith(exts)) ) { if (alreadySynced.includes(path)) { @@ -391,6 +396,11 @@ export async function handleFile( typed.codebase = await codebase.getDigest(forceTar); } + // Scan for __module/ folder alongside the script + const scriptBasePath = path.substring(0, path.indexOf(".")); + const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(); + const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs); + const requestBodyCommon: NewScript = { content, description: typed?.description ?? "", @@ -419,6 +429,7 @@ export async function handleFile( timeout: typed?.timeout, on_behalf_of_email: typed?.on_behalf_of_email, envs: typed?.envs, + modules: modules, }; // console.log(requestBodyCommon.codebase); @@ -460,7 +471,8 @@ export async function handleFile( typed.debounce_delay_s == remote["debounce_delay_s"] && typed.codebase == remote.codebase && typed.on_behalf_of_email == remote.on_behalf_of_email && - deepEqual(typed.envs, remote.envs)) + deepEqual(typed.envs, remote.envs) && + deepEqual(modules ?? null, remote.modules ?? null)) ) { log.info(colors.green(`Script ${remotePath} is up to date`)); return true; @@ -506,6 +518,96 @@ export async function handleFile( return false; } +/** + * Read module files from a __module/ directory on disk. + * Returns the modules record for the API, or undefined if no module folder exists. + */ +async function readModulesFromDisk( + moduleFolderPath: string, + defaultTs: "bun" | "deno" | undefined +): Promise | undefined> { + if (!fs.existsSync(moduleFolderPath) || !fs.statSync(moduleFolderPath).isDirectory()) { + return undefined; + } + + const modules: Record = {}; + + function readDir(dirPath: string, relPrefix: string) { + const entries = fs.readdirSync(dirPath, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dirPath, entry.name); + const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name; + + if (entry.isDirectory()) { + readDir(fullPath, relPath); + } else if (entry.isFile() && !entry.name.endsWith(".script.lock")) { + // Skip lock files — they're handled as the `lock` field on ScriptModule + if (exts.some((ext) => entry.name.endsWith(ext))) { + const content = fs.readFileSync(fullPath, "utf-8"); + const language = inferContentTypeFromFilePath(entry.name, defaultTs); + + // Check for an accompanying lock file + const lockPath = fullPath.replace(/\.[^.]+$/, ".script.lock"); + let lock: string | undefined; + // For multi-extension files (e.g., .pg.sql), try removing from first dot + const baseName = entry.name.substring(0, entry.name.indexOf(".")); + const lockPath2 = path.join(dirPath, baseName + ".script.lock"); + if (fs.existsSync(lockPath)) { + lock = fs.readFileSync(lockPath, "utf-8"); + } else if (fs.existsSync(lockPath2)) { + lock = fs.readFileSync(lockPath2, "utf-8"); + } + + modules[relPath] = { + content, + language: language as ScriptModule["language"], + lock: lock || undefined, + }; + } + } + } + } + + readDir(moduleFolderPath, ""); + + if (Object.keys(modules).length === 0) { + return undefined; + } + + log.debug(`Found ${Object.keys(modules).length} module(s) in ${moduleFolderPath}`); + return modules; +} + +/** + * Write module files to a __module/ directory on disk during pull. + */ +export async function writeModulesToDisk( + moduleFolderPath: string, + modules: Record, + defaultTs: "bun" | "deno" | undefined +): Promise { + // Ensure the module folder exists + fs.mkdirSync(moduleFolderPath, { recursive: true }); + + for (const [relPath, mod] of Object.entries(modules)) { + const fullPath = path.join(moduleFolderPath, relPath); + const dir = path.dirname(fullPath); + fs.mkdirSync(dir, { recursive: true }); + + // Write the module content + fs.writeFileSync(fullPath, mod.content, "utf-8"); + + // Write the lock file if present + if (mod.lock) { + const baseName = relPath.substring(0, relPath.indexOf(".")); + const lockPath = path.join(moduleFolderPath, baseName + ".script.lock"); + const lockDir = path.dirname(lockPath); + fs.mkdirSync(lockDir, { recursive: true }); + fs.writeFileSync(lockPath, mod.lock, "utf-8"); + } + } +} + async function createScript( bundleContent: string | Blob | undefined, workspaceId: string, diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 7d48a14848168..1f18c6c2bb8d2 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -31,6 +31,7 @@ import { findResourceFile, handleScriptMetadata, removeExtensionToPath, + filePathExtensionFromContentType, } from "../script/script.ts"; import { handleFile } from "../script/script.ts"; @@ -68,7 +69,7 @@ import { readLockfile, workspaceDependenciesPathToLanguageAndFilename, } from "../../utils/metadata.ts"; -import { OpenFlow, NativeServiceName } from "../../../gen/types.gen.ts"; +import { OpenFlow, NativeServiceName, ScriptModule } from "../../../gen/types.gen.ts"; import { pushResource } from "../resource/resource.ts"; import { newPathAssigner, @@ -97,6 +98,8 @@ import { getFolderSuffix, getFolderSuffixWithSep, getNonDottedPaths, + isScriptModulePath, + getModuleFolderSuffix, } from "../../utils/resource_folders.ts"; // Merge CLI options with effective settings, preserving CLI flags as overrides @@ -889,6 +892,8 @@ function ZipFSElement( if (ignoreCodebaseChanges && parsed["codebase"]) { parsed["codebase"] = undefined; } + // Modules are stored as files in __module/ folder, not in metadata + delete parsed["modules"]; return useYaml ? yamlStringify(parsed, yamlOptions) : JSON.stringify(parsed, null, 2); @@ -958,6 +963,51 @@ function ZipFSElement( }, }); } + + // Extract script modules into __module/ folder + const scriptModules: Record | undefined = parsed["modules"]; + if (scriptModules && Object.keys(scriptModules).length > 0) { + // The script metadata base path (without .script.json/.script.yaml) + const scriptBasePath = removeSuffix( + removeSuffix(finalPath, ".json"), + ".script" + ); + const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(); + + r.push({ + isDirectory: true, + path: moduleFolderPath, + async *getChildren() { + for (const [relPath, mod] of Object.entries(scriptModules)) { + // Yield the module content file + yield { + isDirectory: false, + path: path.join(moduleFolderPath, relPath), + async *getChildren() {}, + async getContentText() { + return mod.content; + }, + }; + + // Yield the module lock file if present + if (mod.lock) { + const baseName = relPath.substring(0, relPath.indexOf(".")); + yield { + isDirectory: false, + path: path.join(moduleFolderPath, baseName + ".script.lock"), + async *getChildren() {}, + async getContentText() { + return mod.lock!; + }, + }; + } + } + }, + async getContentText() { + throw new Error("Cannot get content of directory"); + }, + }); + } } if (kind == "resource") { const content = await f.async("text"); @@ -1133,6 +1183,12 @@ export async function elementsToMap( continue; } const path = entry.path; + // Include module files in the map so they're compared for changes, + // but they're pushed as part of their parent script via handleFile + if (isScriptModulePath(path)) { + map[path] = await entry.getContentText(); + continue; + } if ( !isFileResource(path) && !isFilesetResource(path) && @@ -1723,6 +1779,21 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { if (!tracker.rawApps.includes(folder)) { tracker.rawApps.push(folder); } + } else if (isScriptModulePath(p)) { + // Module file changed — find the parent script content file + const moduleSuffix = getModuleFolderSuffix() + "/"; + const idx = p.indexOf(moduleSuffix); + if (idx !== -1) { + const scriptBasePath = p.substring(0, idx); + try { + const contentPath = await findContentFile(scriptBasePath + ".script.yaml"); + if (contentPath && !tracker.scripts.includes(contentPath)) { + tracker.scripts.push(contentPath); + } + } catch { + // ignore — content file not found + } + } } else { if (!tracker.scripts.includes(p)) { tracker.scripts.push(p); @@ -1755,6 +1826,43 @@ async function buildTracker(changes: Change[]) { return tracker; } +/** + * When a module file changes, find and push the parent script. + * The parent script's handleFile will read the __module/ folder and include all modules. + */ +async function pushParentScriptForModule( + modulePath: string, + workspace: Workspace, + alreadySynced: string[], + message: string | undefined, + opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined, + rawWorkspaceDependencies: Record, + codebases: SyncCodebase[], +): Promise { + const moduleSuffix = getModuleFolderSuffix() + "/"; + const idx = modulePath.indexOf(moduleSuffix); + if (idx === -1) return; + const scriptBasePath = modulePath.substring(0, idx); + + // Find the parent script content file + try { + const contentPath = await findContentFile(scriptBasePath + ".script.yaml"); + if (contentPath) { + await handleFile( + contentPath, + workspace, + alreadySynced, + message, + opts, + rawWorkspaceDependencies, + codebases, + ); + } + } catch { + log.debug(`Could not find parent script for module: ${modulePath}`); + } +} + export async function pull( opts: GlobalOptions & SyncOptions & { repository?: string; promotion?: string; branch?: string }, @@ -2683,6 +2791,21 @@ export async function push( await writeFile(stateTarget, change.after, "utf-8"); } continue; + } else if (isScriptModulePath(change.path)) { + // Module file changed — push the parent script + await pushParentScriptForModule( + change.path, + workspace, + alreadySynced, + opts.message, + opts, + rawWorkspaceDependencies, + codebases, + ); + if (stateTarget) { + await writeFile(stateTarget, change.after, "utf-8"); + } + continue; } if (stateTarget) { await mkdir(path.dirname(stateTarget), { recursive: true }); @@ -2807,6 +2930,17 @@ export async function push( ) ) { continue; + } else if (isScriptModulePath(change.path)) { + await pushParentScriptForModule( + change.path, + workspace, + alreadySynced, + opts.message, + opts, + rawWorkspaceDependencies, + codebases, + ); + continue; } if (stateTarget) { await mkdir(path.dirname(stateTarget), { recursive: true }); @@ -2848,6 +2982,19 @@ export async function push( if (change.path.endsWith(".lock")) { continue; } + if (isScriptModulePath(change.path)) { + // Module file deleted — push the parent script (which will now have fewer modules) + await pushParentScriptForModule( + change.path, + workspace, + alreadySynced, + opts.message, + opts, + rawWorkspaceDependencies, + codebases, + ); + continue; + } const typ = getTypeStrFromPath(change.path); if (typ == "script") { diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 590c3d7d8e0e5..c73ccb9eb69d9 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -5,7 +5,8 @@ import * as log from "../core/log.ts"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "./yaml.ts"; import { readFile, writeFile, stat, rm, readdir } from "node:fs/promises"; -import { readFileSync } from "node:fs"; +import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs"; +import * as path from "node:path"; import { createRequire } from "node:module"; import { ScriptMetadata, @@ -15,8 +16,10 @@ import { Workspace } from "../commands/workspace/workspace.ts"; import { ScriptLanguage, workspaceDependenciesLanguages, + languageNeedsLock, } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; +import { getModuleFolderSuffix } from "./resource_folders.ts"; import { findCodebase, yamlOptions } from "../commands/sync/sync.ts"; import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts"; @@ -256,6 +259,13 @@ export async function generateScriptMetadataInternal( } else { metadataParsedContent.lock = ""; } + + // Generate locks for modules in __module/ folder + const scriptBasePath = scriptPath.substring(0, scriptPath.indexOf(".")); + const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(); + if (existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory()) { + await updateModuleLocks(workspace, moduleFolderPath, "", remotePath, rawWorkspaceDependencies, opts.defaultTs); + } } else { metadataParsedContent.lock = "!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock"; @@ -545,6 +555,71 @@ async function updateScriptLock( } } +/** + * Generate locks for all module files in a __module/ directory. + * Recursively walks the directory and generates a lock for each module + * whose language requires one. + */ +async function updateModuleLocks( + workspace: Workspace, + dirPath: string, + relPrefix: string, + scriptRemotePath: string, + rawWorkspaceDependencies: Record, + defaultTs: "bun" | "deno" | undefined, +): Promise { + const entries = readdirSync(dirPath, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dirPath, entry.name); + const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name; + + if (entry.isDirectory()) { + await updateModuleLocks(workspace, fullPath, relPath, scriptRemotePath, rawWorkspaceDependencies, defaultTs); + } else if (entry.isFile() && !entry.name.endsWith(".script.lock")) { + let modLanguage: ScriptLanguage; + try { + modLanguage = inferContentTypeFromFilePath(entry.name, defaultTs); + } catch { + continue; // skip files with unrecognized extensions + } + + if (!languageNeedsLock(modLanguage)) continue; + + const moduleContent = readFileSync(fullPath, "utf-8"); + const moduleRemotePath = scriptRemotePath + "/" + relPath; + + log.info(colors.gray(`Generating lock for module ${relPath}`)); + + try { + const lock = await fetchScriptLock( + workspace, + moduleContent, + modLanguage, + moduleRemotePath, + rawWorkspaceDependencies, + ); + + const baseName = entry.name.substring(0, entry.name.indexOf(".")); + const lockPath = path.join(dirPath, baseName + ".script.lock"); + if (lock != "") { + writeFileSync(lockPath, lock, "utf-8"); + } else { + try { + if (existsSync(lockPath)) { + const { rm: rmAsync } = await import("node:fs/promises"); + await rmAsync(lockPath); + } + } catch { + // ignore + } + } + } catch (e) { + log.info(colors.yellow(`Failed to generate lock for module ${relPath}: ${e}`)); + } + } + } +} + //////////////////////////////////////////////////////////////////////////////////////////// // below functions copied from Windmill's FE inferArgs function. TODO: refactor // //////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index e24835ab35fcd..082c9d1fc1edd 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -433,6 +433,40 @@ export function isRawAppFolderMetadataFile(p: string): boolean { ); } +// ============================================================================ +// Script Module Path Functions +// ============================================================================ + +/** + * The suffix used for script module folders. + * Unlike flows/apps, modules always use `__module` (never dotted `.module`) + * to avoid confusion with file extensions. + */ +const MODULE_SUFFIX = "__module"; + +/** + * Get the module folder suffix (always "__module") + */ +export function getModuleFolderSuffix(): string { + return MODULE_SUFFIX; +} + +/** + * Check if a path is inside a script module folder. + * Matches patterns like: .../my_script__module/... + */ +export function isScriptModulePath(p: string): boolean { + return normalizeSep(p).includes(MODULE_SUFFIX + "/"); +} + +/** + * Build the module folder path from a script's base path (without extension). + * e.g., "f/my_script" -> "f/my_script__module" + */ +export function buildModuleFolderPath(scriptBasePath: string): string { + return scriptBasePath + MODULE_SUFFIX; +} + // ============================================================================ // Sync-related Path Functions // ============================================================================ diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index e4e166791b2e7..5e630d4a2d037 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -402,6 +402,7 @@ } async function handleEditScript(stay: boolean, deployMsg?: string): Promise { + scriptEditor?.flushModuleState() // Fetch latest version and fetch entire script after if needed let actual_parent_hash: string | undefined = undefined @@ -558,7 +559,8 @@ deployment_message: deploymentMsg || undefined, on_behalf_of_email: script.on_behalf_of_email, preserve_on_behalf_of: preserveOnBehalfOf || undefined, - assets: script.assets + assets: script.assets, + modules: script.modules } }) @@ -605,6 +607,7 @@ } async function saveDraft(forceSave = false): Promise { + scriptEditor?.flushModuleState() if (initialPath != '' && !savedScript) { return } @@ -709,7 +712,8 @@ no_main_func: script.no_main_func, has_preprocessor: script.has_preprocessor, on_behalf_of_email: script.on_behalf_of_email, - assets: script.assets + assets: script.assets, + modules: script.modules } }) } @@ -1947,6 +1951,7 @@ bind:hasPreprocessor bind:captureTable bind:assets={script.assets} + bind:modules={script.modules} enablePreprocessorSnippet /> diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index f310010c1803e..99c7298e81ac8 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -2,7 +2,7 @@ import { BROWSER } from 'esm-env' import type { Schema, SupportedLanguage } from '$lib/common' - import { type CompletedJob, type Job, JobService, type Preview, type ScriptLang } from '$lib/gen' + import { type CompletedJob, type Job, JobService, type Preview, type ScriptLang, type ScriptModule } from '$lib/gen' import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import { copyToClipboard, @@ -25,6 +25,7 @@ import WindmillIcon from './icons/WindmillIcon.svelte' import * as Y from 'yjs' import { scriptLangToEditorLang } from '$lib/scripts' + import { langToExt } from '$lib/editorLangUtils' import { WebsocketProvider } from 'y-websocket' import Modal from './common/modal/Modal.svelte' import DiffEditor from './DiffEditor.svelte' @@ -40,8 +41,10 @@ GitBranch, Play, PlayIcon, + Plus, Terminal, - WandSparkles + WandSparkles, + X } from 'lucide-svelte' import { DebugToolbar, @@ -123,6 +126,7 @@ lastDeployedCode?: string | undefined disableAi?: boolean assets?: AssetWithAltAccessType[] + modules?: { [key: string]: ScriptModule } | null editorBarRight?: import('svelte').Snippet enablePreprocessorSnippet?: boolean } @@ -155,6 +159,7 @@ lastDeployedCode = undefined, disableAi = false, assets = $bindable(), + modules = $bindable(undefined), editorBarRight, enablePreprocessorSnippet = false }: Props = $props() @@ -163,6 +168,109 @@ let jsonView = $state(false) let schemaHeight = $state(0) + // Module tab state + let activeModuleTab: string | null = $state(null) + let mainCodeBackup: string | null = $state(null) + + function switchToModule(modulePath: string) { + if (activeModuleTab === null) { + // Switching from main: save main code + mainCodeBackup = code + } else if (modules && activeModuleTab !== modulePath) { + // Switching from another module: save its content + modules[activeModuleTab] = { ...modules[activeModuleTab], content: code } + } + if (modules && modules[modulePath]) { + activeModuleTab = modulePath + code = modules[modulePath].content + editor?.setCode(code) + } + } + + function switchToMain() { + if (activeModuleTab !== null && modules) { + // Save current module content + modules[activeModuleTab] = { ...modules[activeModuleTab], content: code } + } + activeModuleTab = null + if (mainCodeBackup !== null) { + code = mainCodeBackup + mainCodeBackup = null + editor?.setCode(code) + } + } + + let effectiveLang = $derived( + activeModuleTab && modules?.[activeModuleTab] + ? (modules[activeModuleTab].language as Preview['language']) + : lang + ) + + let supportsModules = $derived(lang === 'bun' || lang === 'python3') + let mainFileName = $derived('main.' + langToExt(scriptLangToEditorLang(lang))) + + let modulePathInput = $state('') + let showAddModuleDialog = $state(false) + + function inferModuleLang(filePath: string): ScriptModule['language'] { + if (filePath.endsWith('.py')) return 'python3' + if (filePath.endsWith('.go')) return 'go' + if (filePath.endsWith('.sh')) return 'bash' + if (filePath.endsWith('.ps1')) return 'powershell' + if (filePath.endsWith('.php')) return 'php' + if (filePath.endsWith('.rs')) return 'rust' + if (filePath.endsWith('.cs')) return 'csharp' + if (filePath.endsWith('.rb')) return 'ruby' + if (filePath.endsWith('.java')) return 'java' + if (filePath.endsWith('.nu')) return 'nu' + if (filePath.endsWith('.ts')) return (lang === 'deno' ? 'deno' : 'bun') as ScriptModule['language'] + if (filePath.endsWith('.js')) return 'bun' + if (filePath.endsWith('.pg.sql')) return 'postgresql' + if (filePath.endsWith('.my.sql')) return 'mysql' + if (filePath.endsWith('.bq.sql')) return 'bigquery' + if (filePath.endsWith('.sf.sql')) return 'snowflake' + if (filePath.endsWith('.ms.sql')) return 'mssql' + if (filePath.endsWith('.gql')) return 'graphql' + return lang as ScriptModule['language'] + } + + function addModule() { + if (!modulePathInput.trim()) return + const modulePath = modulePathInput.trim() + if (!modules) { + modules = {} + } + if (modules[modulePath]) { + sendUserToast(`Module ${modulePath} already exists`, true) + return + } + modules[modulePath] = { + content: '', + language: inferModuleLang(modulePath) + } + modulePathInput = '' + showAddModuleDialog = false + switchToModule(modulePath) + } + + function removeModule(modulePath: string) { + if (!modules) return + if (activeModuleTab === modulePath) { + switchToMain() + } + delete modules[modulePath] + modules = { ...modules } + } + + export function flushModuleState() { + if (activeModuleTab !== null && modules) { + modules[activeModuleTab] = { ...modules[activeModuleTab], content: code } + code = mainCodeBackup ?? code + activeModuleTab = null + mainCodeBackup = null + } + } + $effect.pre(() => { if (schema == undefined) { schema = emptySchema() @@ -1317,8 +1425,53 @@ {#snippet editorContent()} -
+
+ {#if supportsModules && modules && Object.keys(modules).length > 0} +
+ + {#each Object.keys(modules) as modulePath} + + {/each} + +
+ {/if} +
+ {#if supportsModules && (!modules || Object.keys(modules).length === 0)} + + {/if} {#if assets?.length} {/if} @@ -1435,10 +1588,11 @@
{/if}
+
{/snippet} {#snippet editorPane()} - {#key lang} + {#key effectiveLang} { - inferSchema(e.detail) + if (activeModuleTab === null) { + inferSchema(e.detail) + } // Refresh breakpoint positions when code changes (decorations track their lines) if (debugMode && breakpointDecorations.length > 0) { refreshBreakpointPositions() @@ -1458,11 +1614,15 @@ on:saveDraft on:toggleTestPanel={toggleTestPanel} cmdEnterAction={async () => { - await inferSchema(code) + if (activeModuleTab === null) { + await inferSchema(code) + } runTest() }} formatAction={async () => { - await inferSchema(code) + if (activeModuleTab === null) { + await inferSchema(code) + } try { localStorage.setItem(path ?? 'last_save', code) } catch (e) { @@ -1471,7 +1631,7 @@ dispatch('format') }} class="flex flex-1 h-full !overflow-visible" - scriptLang={lang} + scriptLang={effectiveLang} automaticLayout={true} {fixedOverflowWidgets} {args} @@ -1518,6 +1678,25 @@ on:addInventories={handleAddInventories} /> +{#if showAddModuleDialog} + +
+ + { if (e.key === 'Enter') addModule() }} + /> +
+ + +
+
+ +{/if} +