From 2e5ba3ebf50cfa18df216d41bafe1e8a34fc20fb Mon Sep 17 00:00:00 2001 From: queil Date: Wed, 6 May 2026 08:46:59 +0200 Subject: [PATCH 1/7] wip: subpath impl --- examples/volume-subpath.yaml | 8 ++++++ src/api/container.rs | 50 +----------------------------------- src/api/sidecar.rs | 28 +++----------------- src/api/volume.rs | 43 ++++++++++++++++++++++++++----- src/api/workspace/create.rs | 12 +-------- src/cmd/new.rs | 6 ++--- 6 files changed, 54 insertions(+), 93 deletions(-) create mode 100644 examples/volume-subpath.yaml diff --git a/examples/volume-subpath.yaml b/examples/volume-subpath.yaml new file mode 100644 index 0000000..6f69788 --- /dev/null +++ b/examples/volume-subpath.yaml @@ -0,0 +1,8 @@ +data: + git-config: + content: |- + [url "git@github.com:"] + insteadOf = gh: + +mounts: + ~/.gitconfig: git-config diff --git a/src/api/container.rs b/src/api/container.rs index cb18f5c..c02ad05 100644 --- a/src/api/container.rs +++ b/src/api/container.rs @@ -11,7 +11,6 @@ use crate::{ use base64::{Engine as _, engine::general_purpose}; use bollard::{ - body_full, errors::Error::{self, DockerResponseServerError}, models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerState, @@ -25,12 +24,9 @@ use bollard::{ }, }; -use crate::model::types::{TargetDir, VolumeFilesSpec}; - use bollard_stubs::models::{MountType, NetworkingConfig}; -use bollard_stubs::query_parameters::{UploadToContainerOptions, WaitContainerOptions}; +use bollard_stubs::query_parameters::WaitContainerOptions; use futures::{StreamExt, TryStreamExt, future}; -use std::time::{SystemTime, UNIX_EPOCH}; use std::{collections::HashMap, time::Duration}; use tokio::time::{sleep, timeout}; @@ -442,50 +438,6 @@ impl<'a> ContainerApi<'a> { Ok(container_id.clone()) } - pub async fn symlink_files( - &self, - container_id: &str, - mounts: &HashMap, - uid: Option, - ) -> Result<(), AnyError> { - let mut archive = tar::Builder::new(Vec::new()); - for (_, spec) in mounts { - for file in &spec.files { - log::debug!( - "Creating symlink: {} -> {}", - &file.user_file.as_str(), - &file.target_file.as_str() - ); - let mut header = tar::Header::new_gnu(); - header.set_size(0); - header.set_mode(0o777); - header.set_uid(uid.unwrap_or(constants::ROOT_UID_INT) as u64); - header.set_gid(uid.unwrap_or(constants::ROOT_UID_INT) as u64); - header.set_entry_type(tar::EntryType::Symlink); - header.set_mtime(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()); - archive.append_link( - &mut header, - file.user_file.as_str().trim_start_matches("/"), - file.target_file.as_str(), - )?; - } - } - let tar_bytes = archive.into_inner()?; - - self.client - .upload_to_container( - &container_id, - Some(UploadToContainerOptions { - path: "/".to_string(), - ..Default::default() - }), - body_full(tar_bytes.into()), - ) - .await?; - - Ok(()) - } - pub async fn start(&self, container_id: &str) -> Result<(), Error> { self.client .start_container(&container_id, None::) diff --git a/src/api/sidecar.rs b/src/api/sidecar.rs index 2763057..9e3cdaf 100644 --- a/src/api/sidecar.rs +++ b/src/api/sidecar.rs @@ -69,7 +69,7 @@ impl<'a> WorkspaceApi<'a> { s.real_mounts.insert(t.clone(), m.clone()); // The volume might already be created by the workspace-level volume creation // but still may need files in the paths not covered by that process - self.api.volume.populate_volume(t, m, uid).await?; + self.api.volume.populate_volume(m, uid).await?; } let cmd = &s.command.iter().map(|x| x.as_str()).collect::>(); @@ -137,12 +137,6 @@ impl<'a> WorkspaceApi<'a> { }) .await? { - // IMPORTANT: make symlinks to the mounted volumes before install so the - // mounted files/dirs are available - self.api - .container - .symlink_files(&container_id, &real_mounts, uid) - .await?; self.api.container.start(&container_id).await?; self.api .exec @@ -180,29 +174,15 @@ impl<'a> WorkspaceApi<'a> { } let latest_runtime_image = format!("{}:latest", runtime_image); - if let ContainerResult::Created { id: container_id } = self - .api + self.api .container .create(RunSpec { image: &latest_runtime_image, ..run_spec.clone() }) - .await? - { - self.api - .container - .symlink_files(&container_id, &real_mounts, uid) - .await?; - } + .await?; } else { - if let ContainerResult::Created { id: container_id } = - self.api.container.create(run_spec).await? - { - self.api - .container - .symlink_files(&container_id, &real_mounts, uid) - .await?; - } + self.api.container.create(run_spec).await?; } } diff --git a/src/api/volume.rs b/src/api/volume.rs index a8981c8..297618d 100644 --- a/src/api/volume.rs +++ b/src/api/volume.rs @@ -20,13 +20,15 @@ use crate::{ use base64::{Engine as _, engine::general_purpose}; use bollard::{ errors::Error::DockerResponseServerError, - models::Volume, + models::{MountVolumeOptions, Volume}, query_parameters::{ListVolumesOptions, RemoveVolumeOptions}, service::Mount, }; use bollard_stubs::models::MountType::VOLUME; use bollard_stubs::models::VolumeCreateRequest; +const SHADOW_ROOT_DIR: &str = "/var/lib/rooz"; + impl<'a> VolumeApi<'a> { pub async fn get_all(&self, labels: &Labels) -> Result, AnyError> { let list_options = ListVolumesOptions { @@ -249,7 +251,6 @@ impl<'a> VolumeApi<'a> { mounts: HashMap, home_dir: Option<&str>, ) -> HashMap { - const SHADOW_ROOT_DIR: &str = "/var/lib/rooz"; mounts .iter() .map(|(target, source_entry)| { @@ -265,7 +266,7 @@ impl<'a> VolumeApi<'a> { .with_extension("data"); ( - SHADOW_ROOT_DIR.to_string(), + expanded_target.clone(), Some(FileSpec { target_file: TargetFile(shadow_file.to_string_lossy().to_string()), user_file: UserFile(expanded_target), @@ -314,20 +315,49 @@ impl<'a> VolumeApi<'a> { pub async fn populate_volume( &self, - target_dir: TargetDir, volume_file: VolumeFilesSpec, uid: Option, ) -> Result<(), AnyError> { + let populate_target = TargetDir(SHADOW_ROOT_DIR.to_string()); self.ensure_file_v2( - target_dir.as_str(), + SHADOW_ROOT_DIR, &volume_file.clone(), - Self::mount(&target_dir, &volume_file), + Self::populate_mount(&populate_target, &volume_file), uid, ) .await } pub fn mount(target: &TargetDir, source: &VolumeFilesSpec) -> Mount { + debug_assert!( + source.files.len() <= 1, + "user-container mount expects 0 or 1 file per target; got {} for {}", + source.files.len(), + target.as_str() + ); + + let subpath = source.files.first().map(|f| { + Path::new(f.target_file.as_str()) + .file_name() + .unwrap() + .to_string_lossy() + .into_owned() + }); + + Mount { + target: Some(target.as_str().to_string()), + source: Some(source.volume_name.as_str().to_string()), + typ: Some(VOLUME), + read_only: Some(subpath.is_some()), + volume_options: subpath.map(|sp| MountVolumeOptions { + subpath: Some(sp), + ..Default::default() + }), + ..Mount::default() + } + } + + pub fn populate_mount(target: &TargetDir, source: &VolumeFilesSpec) -> Mount { Mount { target: Some(target.as_str().to_string()), source: Some(source.volume_name.as_str().to_string()), @@ -336,6 +366,7 @@ impl<'a> VolumeApi<'a> { ..Mount::default() } } + pub async fn mounts_v2( &self, real_mounts: &HashMap, diff --git a/src/api/workspace/create.rs b/src/api/workspace/create.rs index 8913468..234761a 100644 --- a/src/api/workspace/create.rs +++ b/src/api/workspace/create.rs @@ -1,4 +1,3 @@ -use crate::model::types::{TargetDir, VolumeFilesSpec}; use crate::{ api::WorkspaceApi, model::{ @@ -7,15 +6,10 @@ use crate::{ }, util::ssh, }; -use std::collections::HashMap; use std::path::Path; impl<'a> WorkspaceApi<'a> { - pub async fn create( - &self, - spec: &WorkSpec<'a>, - real_mounts: &HashMap, - ) -> Result { + pub async fn create(&self, spec: &WorkSpec<'a>) -> Result { let mut volumes = vec![]; if let Some(caches) = &spec.caches { @@ -76,10 +70,6 @@ impl<'a> WorkspaceApi<'a> { match self.api.container.create(run_spec).await? { ContainerResult::Created { id: container_id } => { - self.api - .container - .symlink_files(&container_id, &real_mounts, Some(spec.uid.parse::()?)) - .await?; if let Some(install) = spec.install.clone() { self.api.container.start(&container_id).await?; self.api diff --git a/src/cmd/new.rs b/src/cmd/new.rs index b2ce1ab..1944fd1 100644 --- a/src/cmd/new.rs +++ b/src/cmd/new.rs @@ -79,13 +79,13 @@ impl<'a> WorkspaceApi<'a> { let mut cfg2 = cfg.clone(); let mounts_v2 = self.api.volume.mounts_v2(&real_mounts).await?; - for (t, m) in real_mounts.clone() { + for (_, m) in real_mounts.clone() { //TODO: when initializing volumes both here in sidecars we should verify // if each file exists and if not create them if let VolumeResult::Created {} = volume_results[&m.volume_name] { self.api .volume - .populate_volume(t, m, Some(work_spec.uid.to_string().parse::()?)) + .populate_volume(m, Some(work_spec.uid.to_string().parse::()?)) .await?; } } @@ -184,7 +184,7 @@ impl<'a> WorkspaceApi<'a> { ..*work_spec }; - let ws = self.create(&work_spec, &real_mounts).await?; + let ws = self.create(&work_spec).await?; if !cfg2.extra_repos.is_empty() { self.git .clone_extra_repos(clone_spec.clone(), cfg2.extra_repos) From e1f6bdd65dfe93d9360422eb117e2e00cbb5c1c4 Mon Sep 17 00:00:00 2001 From: queil <4584075+queil@users.noreply.github.com> Date: Wed, 6 May 2026 07:10:10 +0000 Subject: [PATCH 2/7] tweak --- examples/volume-subpath.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/volume-subpath.yaml b/examples/volume-subpath.yaml index 6f69788..469e557 100644 --- a/examples/volume-subpath.yaml +++ b/examples/volume-subpath.yaml @@ -4,5 +4,8 @@ data: [url "git@github.com:"] insteadOf = gh: +image: docker.io/alpine +shell: ["sh"] + mounts: ~/.gitconfig: git-config From f55f3588e3ddc4dd5b56aaa64c735625c7f13aee Mon Sep 17 00:00:00 2001 From: queil Date: Wed, 6 May 2026 12:17:18 +0200 Subject: [PATCH 3/7] fix: use no-copy to make subpath work --- src/api/volume.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api/volume.rs b/src/api/volume.rs index 297618d..580f00b 100644 --- a/src/api/volume.rs +++ b/src/api/volume.rs @@ -351,6 +351,10 @@ impl<'a> VolumeApi<'a> { read_only: Some(subpath.is_some()), volume_options: subpath.map(|sp| MountVolumeOptions { subpath: Some(sp), + // IMPORTANT: no copy prevents it from failing if the file already exists in the image + // error: Error response from daemon: open /var/lib/docker/tmp/safe-mountXXXXXXXXX: not a directory + // We overlay it anyway so it is not an issue + no_copy: Some(true), ..Default::default() }), ..Mount::default() From 52049d1e941b793da5c87fa848d572e1151f4e81 Mon Sep 17 00:00:00 2001 From: queil Date: Wed, 6 May 2026 12:31:04 +0200 Subject: [PATCH 4/7] fix: do not label git clone container wit workspace key --- src/util/git.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util/git.rs b/src/util/git.rs index 6e1911f..0a56d78 100644 --- a/src/util/git.rs +++ b/src/util/git.rs @@ -135,7 +135,9 @@ impl<'a> GitApi<'a> { } let clone_cmd = container::inject(&clone_script, "clone.sh"); - let labels = Labels::from(&[Labels::workspace(&spec.workspace_key), Labels::role("git")]); + // IMPORTANT: no workspace label here as those do not really belong to workspace + // it will get refactored to use one shot + let labels = Labels::from(&[Labels::role("git")]); let mut mounts = vec![ssh::mount("/tmp/.ssh")]; let mut volumes: Vec = vec![]; From 207d0d63767e4cc10762187648c63f6542870f2f Mon Sep 17 00:00:00 2001 From: queil Date: Wed, 6 May 2026 13:05:06 +0200 Subject: [PATCH 5/7] fix: read_only false, some tools require to update --- src/api/volume.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/volume.rs b/src/api/volume.rs index 580f00b..d392af6 100644 --- a/src/api/volume.rs +++ b/src/api/volume.rs @@ -348,7 +348,7 @@ impl<'a> VolumeApi<'a> { target: Some(target.as_str().to_string()), source: Some(source.volume_name.as_str().to_string()), typ: Some(VOLUME), - read_only: Some(subpath.is_some()), + read_only: Some(false), volume_options: subpath.map(|sp| MountVolumeOptions { subpath: Some(sp), // IMPORTANT: no copy prevents it from failing if the file already exists in the image From 0501b22616724889c19344bc4af5b32ad80b695e Mon Sep 17 00:00:00 2001 From: queil Date: Fri, 8 May 2026 14:33:06 +0200 Subject: [PATCH 6/7] bump --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0d0a0c..fd47ed2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -295,9 +295,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.61" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "shlex", From 69f7d8ac4442ab7289e3c0c9dd75178cdd59cf8e Mon Sep 17 00:00:00 2001 From: queil <4584075+queil@users.noreply.github.com> Date: Mon, 18 May 2026 12:02:25 +0000 Subject: [PATCH 7/7] fix: remove workaround for Docker < 29.5 --- src/api/volume.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/api/volume.rs b/src/api/volume.rs index d392af6..71fa547 100644 --- a/src/api/volume.rs +++ b/src/api/volume.rs @@ -351,10 +351,6 @@ impl<'a> VolumeApi<'a> { read_only: Some(false), volume_options: subpath.map(|sp| MountVolumeOptions { subpath: Some(sp), - // IMPORTANT: no copy prevents it from failing if the file already exists in the image - // error: Error response from daemon: open /var/lib/docker/tmp/safe-mountXXXXXXXXX: not a directory - // We overlay it anyway so it is not an issue - no_copy: Some(true), ..Default::default() }), ..Mount::default()