Vol subpath v2

examples/volume-subpath.yaml

@@ -0,0 +1,11 @@
+data:
+  git-config:
+    content: |-
+      [url "git@github.com:"]
+      insteadOf = gh:
+
+image: docker.io/alpine
+shell: ["sh"]
+
+mounts:
+  ~/.gitconfig: git-config

src/api/container.rs

@@ -11,7 +11,6 @@ use crate::{
 use base64::{Engine as _, engine::general_purpose};
 
 use bollard::{
-    body_full,
     errors::Error::{self, DockerResponseServerError},
     models::{
         ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerState,
@@ -25,12 +24,9 @@ use bollard::{
     },
 };
 
-use crate::model::types::{TargetDir, VolumeFilesSpec};
-
 use bollard_stubs::models::{MountType, NetworkingConfig};
-use bollard_stubs::query_parameters::{UploadToContainerOptions, WaitContainerOptions};
+use bollard_stubs::query_parameters::WaitContainerOptions;
 use futures::{StreamExt, TryStreamExt, future};
-use std::time::{SystemTime, UNIX_EPOCH};
 use std::{collections::HashMap, time::Duration};
 use tokio::time::{sleep, timeout};
 
@@ -442,50 +438,6 @@ impl<'a> ContainerApi<'a> {
         Ok(container_id.clone())
     }
 
-    pub async fn symlink_files(
-        &self,
-        container_id: &str,
-        mounts: &HashMap<TargetDir, VolumeFilesSpec>,
-        uid: Option<i32>,
-    ) -> Result<(), AnyError> {
-        let mut archive = tar::Builder::new(Vec::new());
-        for (_, spec) in mounts {
-            for file in &spec.files {
-                log::debug!(
-                    "Creating symlink: {} -> {}",
-                    &file.user_file.as_str(),
-                    &file.target_file.as_str()
-                );
-                let mut header = tar::Header::new_gnu();
-                header.set_size(0);
-                header.set_mode(0o777);
-                header.set_uid(uid.unwrap_or(constants::ROOT_UID_INT) as u64);
-                header.set_gid(uid.unwrap_or(constants::ROOT_UID_INT) as u64);
-                header.set_entry_type(tar::EntryType::Symlink);
-                header.set_mtime(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs());
-                archive.append_link(
-                    &mut header,
-                    file.user_file.as_str().trim_start_matches("/"),
-                    file.target_file.as_str(),
-                )?;
-            }
-        }
-        let tar_bytes = archive.into_inner()?;
-
-        self.client
-            .upload_to_container(
-                &container_id,
-                Some(UploadToContainerOptions {
-                    path: "/".to_string(),
-                    ..Default::default()
-                }),
-                body_full(tar_bytes.into()),
-            )
-            .await?;
-
-        Ok(())
-    }
-
     pub async fn start(&self, container_id: &str) -> Result<(), Error> {
         self.client
             .start_container(&container_id, None::<StartContainerOptions>)

src/api/sidecar.rs

@@ -69,7 +69,7 @@ impl<'a> WorkspaceApi<'a> {
                 s.real_mounts.insert(t.clone(), m.clone());
                 // The volume might already be created by the workspace-level volume creation
                 // but still may need files in the paths not covered by that process
-                self.api.volume.populate_volume(t, m, uid).await?;
+                self.api.volume.populate_volume(m, uid).await?;
             }
 
             let cmd = &s.command.iter().map(|x| x.as_str()).collect::<Vec<_>>();
@@ -137,12 +137,6 @@ impl<'a> WorkspaceApi<'a> {
                         })
                         .await?
                     {
-                        // IMPORTANT: make symlinks to the mounted volumes before install so the
-                        // mounted files/dirs are available
-                        self.api
-                            .container
-                            .symlink_files(&container_id, &real_mounts, uid)
-                            .await?;
                         self.api.container.start(&container_id).await?;
                         self.api
                             .exec
@@ -180,29 +174,15 @@ impl<'a> WorkspaceApi<'a> {
                 }
 
                 let latest_runtime_image = format!("{}:latest", runtime_image);
-                if let ContainerResult::Created { id: container_id } = self
-                    .api
+                self.api
                     .container
                     .create(RunSpec {
                         image: &latest_runtime_image,
                         ..run_spec.clone()
                     })
-                    .await?
-                {
-                    self.api
-                        .container
-                        .symlink_files(&container_id, &real_mounts, uid)
-                        .await?;
-                }
+                    .await?;
             } else {
-                if let ContainerResult::Created { id: container_id } =
-                    self.api.container.create(run_spec).await?
-                {
-                    self.api
-                        .container
-                        .symlink_files(&container_id, &real_mounts, uid)
-                        .await?;
-                }
+                self.api.container.create(run_spec).await?;
             }
         }
 

src/api/volume.rs

@@ -20,13 +20,15 @@ use crate::{
 use base64::{Engine as _, engine::general_purpose};
 use bollard::{
     errors::Error::DockerResponseServerError,
-    models::Volume,
+    models::{MountVolumeOptions, Volume},
     query_parameters::{ListVolumesOptions, RemoveVolumeOptions},
     service::Mount,
 };
 use bollard_stubs::models::MountType::VOLUME;
 use bollard_stubs::models::VolumeCreateRequest;
 
+const SHADOW_ROOT_DIR: &str = "/var/lib/rooz";
+
 impl<'a> VolumeApi<'a> {
     pub async fn get_all(&self, labels: &Labels) -> Result<Vec<Volume>, AnyError> {
         let list_options = ListVolumesOptions {
@@ -249,7 +251,6 @@ impl<'a> VolumeApi<'a> {
         mounts: HashMap<TargetPath, DataEntryVolumeSpec>,
         home_dir: Option<&str>,
     ) -> HashMap<TargetDir, VolumeFilesSpec> {
-        const SHADOW_ROOT_DIR: &str = "/var/lib/rooz";
         mounts
             .iter()
             .map(|(target, source_entry)| {
@@ -265,7 +266,7 @@ impl<'a> VolumeApi<'a> {
                             .with_extension("data");
 
                         (
-                            SHADOW_ROOT_DIR.to_string(),
+                            expanded_target.clone(),
                             Some(FileSpec {
                                 target_file: TargetFile(shadow_file.to_string_lossy().to_string()),
                                 user_file: UserFile(expanded_target),
@@ -314,20 +315,49 @@ impl<'a> VolumeApi<'a> {
 
     pub async fn populate_volume(
         &self,
-        target_dir: TargetDir,
         volume_file: VolumeFilesSpec,
         uid: Option<i32>,
     ) -> Result<(), AnyError> {
+        let populate_target = TargetDir(SHADOW_ROOT_DIR.to_string());
         self.ensure_file_v2(
-            target_dir.as_str(),
+            SHADOW_ROOT_DIR,
             &volume_file.clone(),
-            Self::mount(&target_dir, &volume_file),
+            Self::populate_mount(&populate_target, &volume_file),
             uid,
         )
         .await
     }
 
     pub fn mount(target: &TargetDir, source: &VolumeFilesSpec) -> Mount {
+        debug_assert!(
+            source.files.len() <= 1,
+            "user-container mount expects 0 or 1 file per target; got {} for {}",
+            source.files.len(),
+            target.as_str()
+        );
+
+        let subpath = source.files.first().map(|f| {
+            Path::new(f.target_file.as_str())
+                .file_name()
+                .unwrap()
+                .to_string_lossy()
+                .into_owned()
+        });
+
+        Mount {
+            target: Some(target.as_str().to_string()),
+            source: Some(source.volume_name.as_str().to_string()),
+            typ: Some(VOLUME),
+            read_only: Some(false),
+            volume_options: subpath.map(|sp| MountVolumeOptions {
+                subpath: Some(sp),
+                ..Default::default()
+            }),
+            ..Mount::default()
+        }
+    }
+
+    pub fn populate_mount(target: &TargetDir, source: &VolumeFilesSpec) -> Mount {
         Mount {
             target: Some(target.as_str().to_string()),
             source: Some(source.volume_name.as_str().to_string()),
@@ -336,6 +366,7 @@ impl<'a> VolumeApi<'a> {
             ..Mount::default()
         }
     }
+
     pub async fn mounts_v2(
         &self,
         real_mounts: &HashMap<TargetDir, VolumeFilesSpec>,

src/api/workspace/create.rs

@@ -1,4 +1,3 @@
-use crate::model::types::{TargetDir, VolumeFilesSpec};
 use crate::{
     api::WorkspaceApi,
     model::{
@@ -7,15 +6,10 @@ use crate::{
     },
     util::ssh,
 };
-use std::collections::HashMap;
 use std::path::Path;
 
 impl<'a> WorkspaceApi<'a> {
-    pub async fn create(
-        &self,
-        spec: &WorkSpec<'a>,
-        real_mounts: &HashMap<TargetDir, VolumeFilesSpec>,
-    ) -> Result<WorkspaceResult, AnyError> {
+    pub async fn create(&self, spec: &WorkSpec<'a>) -> Result<WorkspaceResult, AnyError> {
         let mut volumes = vec![];
 
         if let Some(caches) = &spec.caches {
@@ -76,10 +70,6 @@ impl<'a> WorkspaceApi<'a> {
 
         match self.api.container.create(run_spec).await? {
             ContainerResult::Created { id: container_id } => {
-                self.api
-                    .container
-                    .symlink_files(&container_id, &real_mounts, Some(spec.uid.parse::<i32>()?))
-                    .await?;
                 if let Some(install) = spec.install.clone() {
                     self.api.container.start(&container_id).await?;
                     self.api

src/cmd/new.rs

@@ -79,13 +79,13 @@ impl<'a> WorkspaceApi<'a> {
         let mut cfg2 = cfg.clone();
 
         let mounts_v2 = self.api.volume.mounts_v2(&real_mounts).await?;
-        for (t, m) in real_mounts.clone() {
+        for (_, m) in real_mounts.clone() {
             //TODO: when initializing volumes both here in sidecars we should verify
             // if each file exists and if not create them
             if let VolumeResult::Created {} = volume_results[&m.volume_name] {
                 self.api
                     .volume
-                    .populate_volume(t, m, Some(work_spec.uid.to_string().parse::<i32>()?))
+                    .populate_volume(m, Some(work_spec.uid.to_string().parse::<i32>()?))
                     .await?;
             }
         }
@@ -184,7 +184,7 @@ impl<'a> WorkspaceApi<'a> {
             ..*work_spec
         };
 
-        let ws = self.create(&work_spec, &real_mounts).await?;
+        let ws = self.create(&work_spec).await?;
         if !cfg2.extra_repos.is_empty() {
             self.git
                 .clone_extra_repos(clone_spec.clone(), cfg2.extra_repos)

src/util/git.rs

@@ -135,7 +135,9 @@ impl<'a> GitApi<'a> {
         }
 
         let clone_cmd = container::inject(&clone_script, "clone.sh");
-        let labels = Labels::from(&[Labels::workspace(&spec.workspace_key), Labels::role("git")]);
+        // IMPORTANT: no workspace label here as those do not really belong to workspace
+        // it will get refactored to use one shot
+        let labels = Labels::from(&[Labels::role("git")]);
         let mut mounts = vec![ssh::mount("/tmp/.ssh")];
 
         let mut volumes: Vec<RoozVolume> = vec![];