diff --git a/examples/extends/base.yaml b/examples/bases/base.yaml similarity index 61% rename from examples/extends/base.yaml rename to examples/bases/base.yaml index 7ed92f8..6bf4e75 100644 --- a/examples/extends/base.yaml +++ b/examples/bases/base.yaml @@ -1,4 +1,4 @@ image: docker.io/alpine sidecars: mongo: - image: docker.io/mongo \ No newline at end of file + image: docker.io/mongo diff --git a/examples/bases/base2.yaml b/examples/bases/base2.yaml new file mode 100644 index 0000000..2f8bf20 --- /dev/null +++ b/examples/bases/base2.yaml @@ -0,0 +1,6 @@ +mounts: + /test: {} + +sidecars: + docker: + image: docker.io/docker diff --git a/examples/bases/overlay.yaml b/examples/bases/overlay.yaml new file mode 100644 index 0000000..35ec0ac --- /dev/null +++ b/examples/bases/overlay.yaml @@ -0,0 +1,13 @@ +vars: + test-x: 1 + +env: + TEST: '{{ test-x }}' + +sidecars: + sql: + image: docker.io/postgres + +bases: +- base.yaml +- base2.yaml diff --git a/examples/extends/overlay.yaml b/examples/extends/overlay.yaml deleted file mode 100644 index feb795c..0000000 --- a/examples/extends/overlay.yaml +++ /dev/null @@ -1,8 +0,0 @@ -vars: - TEST: 1 - -sidecars: - sql: - image: docker.io/postgres - -extends: base.yaml \ No newline at end of file diff --git a/src/api/config.rs b/src/api/config.rs index f541038..9cb00b0 100644 --- a/src/api/config.rs +++ b/src/api/config.rs @@ -15,6 +15,12 @@ use colored::Colorize; use super::{Api, ConfigApi}; +pub struct ConfigBody { + pub body: String, + pub bases: Option, + pub merged: Option, +} + #[async_trait::async_trait] pub trait ConfigReader { async fn read_file(&self, path: &str) -> Result; @@ -45,13 +51,13 @@ impl<'a> ConfigApi<'a> { Ok(()) } - pub async fn store_extends(&self, workspace_key: &str, body: &str) -> Result<(), AnyError> { + pub async fn store_bases(&self, workspace_key: &str, body: &str) -> Result<(), AnyError> { let config_vol = RoozVolume::config_data( workspace_key, "/etc/rooz", Some( [( - ConfigType::Extends.file_path().to_string(), + ConfigType::Bases.file_path().to_string(), body.to_string(), )] .into_iter() @@ -201,46 +207,53 @@ impl<'a> ConfigApi<'a> { child_path: &str, child: RoozCfg, depth: usize, - ) -> Result { + ) -> Result<(RoozCfg, Vec<(String, RoozCfg)>), AnyError> { + let base_paths = match child.bases.as_ref() { + Some(p) if !p.is_empty() => p.clone(), + _ => return Ok((child, vec![])), + }; + if depth >= Self::MAX_EXTENDS_DEPTH { return Err(format!( - "extends nesting too deep (limit {})", + "bases nesting too deep (limit {})", Self::MAX_EXTENDS_DEPTH ) .into()); } - let extends_path = match child.extends.as_deref() { - Some(p) => p, - None => return Ok(child), - }; - - RoozCfg::validate_extends_path(extends_path)?; + RoozCfg::validate_base_list(&base_paths)?; let parent_dir = std::path::Path::new(child_path) .parent() .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_default(); - let abs_extends = if parent_dir.is_empty() { - extends_path.to_string() - } else { - format!("{}/{}", parent_dir, extends_path) - }; - let base_body = reader.read_file(&abs_extends).await?; - if base_body.is_empty() { - return Err(format!("extends '{}' not found or empty", extends_path).into()); - } + let mut individual_bases: Vec<(String, RoozCfg)> = Vec::new(); + let mut accumulated = RoozCfg::none(); + for base_path in &base_paths { + let abs_path = if parent_dir.is_empty() { + base_path.to_string() + } else { + format!("{}/{}", parent_dir, base_path) + }; - let base_fmt = FileFormat::from_path(extends_path); - let base = RoozCfg::deserialize_config(&base_body, base_fmt)? - .ok_or_else(|| format!("Failed to parse extends '{}': invalid config", extends_path))?; + let base_body = reader.read_file(&abs_path).await?; + if base_body.is_empty() { + return Err(format!("base '{}' not found or empty", base_path).into()); + } - let mut effective_base = - Box::pin(self.resolve_extends_chain(reader, &abs_extends, base, depth + 1)).await?; + let base_fmt = FileFormat::from_path(base_path); + let base = RoozCfg::deserialize_config(&base_body, base_fmt)? + .ok_or_else(|| format!("Failed to parse base '{}': invalid config", base_path))?; - effective_base.from_config(&child); - Ok(effective_base) + let (resolved, _) = + Box::pin(self.resolve_extends_chain(reader, &abs_path, base, depth + 1)).await?; + individual_bases.push((base_path.clone(), resolved.clone())); + accumulated.from_config(&resolved); + } + + accumulated.from_config(&child); + Ok((accumulated, individual_bases)) } pub async fn read_config_body( @@ -249,9 +262,7 @@ impl<'a> ConfigApi<'a> { clone_dir: &str, file_format: FileFormat, exact_path: Option<&str>, - ) -> Result)>, AnyError> { - use crate::config::config::RoozCfg; - + ) -> Result, AnyError> { let file_path = match exact_path { Some(p) => format!("{}/{}", clone_dir, p.to_string()), None => format!("{}/.rooz.{}", clone_dir, file_format.to_string()), @@ -281,19 +292,25 @@ impl<'a> ConfigApi<'a> { if let (Some(_), Some(cfg)) = (exact_path, RoozCfg::deserialize_config(&body, file_format)?) { - if cfg.extends.is_some() { + if cfg.bases.is_some() { let reader = ContainerReader { api: self.api, container_id, }; - let merged = self + let (merged, individual_bases) = self .resolve_extends_chain(&reader, &file_path, cfg, 0) .await?; - return Ok(Some((body, Some(merged.to_string(file_format)?)))); + let bases_yaml = individual_bases + .iter() + .map(|(path, b)| b.to_string(file_format).map(|yaml| format!("# {}\n{}", path, yaml))) + .collect::, _>>()? + .join("\n---\n"); + let bases_storage = if bases_yaml.is_empty() { None } else { Some(bases_yaml) }; + return Ok(Some(ConfigBody { body, bases: bases_storage, merged: Some(merged) })); } } - Ok(Some((body, None))) + Ok(Some(ConfigBody { body, bases: None, merged: None })) } pub async fn try_read_config( @@ -301,12 +318,12 @@ impl<'a> ConfigApi<'a> { container_id: &str, clone_dir: &str, ) -> Result, FileFormat)>, AnyError> { - let rooz_cfg = if let Some((body, extends_body)) = self + let rooz_cfg = if let Some(cb) = self .read_config_body(&container_id, &clone_dir, FileFormat::Yaml, None) .await? { log::debug!("Config file found (YAML)"); - Some((body, extends_body, FileFormat::Yaml)) + Some((cb.body, None, FileFormat::Yaml)) } else { log::debug!("No valid config file found"); None diff --git a/src/api/container.rs b/src/api/container.rs index 64aaebd..d60c541 100644 --- a/src/api/container.rs +++ b/src/api/container.rs @@ -26,7 +26,7 @@ use bollard::{ }; use crate::model::types::{TargetDir, VolumeFilesSpec}; -use crate::util::backend::ContainerBackend; + use bollard_stubs::models::{MountTypeEnum, NetworkingConfig}; use bollard_stubs::query_parameters::{UploadToContainerOptions, WaitContainerOptions}; use futures::{StreamExt, TryStreamExt, future}; diff --git a/src/cmd/config/show.rs b/src/cmd/config/show.rs index 1e2cc66..44a91fe 100644 --- a/src/cmd/config/show.rs +++ b/src/cmd/config/show.rs @@ -54,11 +54,11 @@ impl<'a> ConfigApi<'a> { body }; - let extends_body = self.read(workspace_key, &ConfigType::Extends).await?; - if extends_body.is_empty() { + let bases_body = self.read(workspace_key, &ConfigType::Bases).await?; + if bases_body.is_empty() { Some(body) } else { - Some(format!("{}\n{}\n{}", body, EXTENDS_SEPARATOR, extends_body)) + Some(format!("{}\n{}\n{}", body, EXTENDS_SEPARATOR, bases_body)) } } ConfigPart::Runtime => { diff --git a/src/cmd/new.rs b/src/cmd/new.rs index a9710b9..90fbbac 100644 --- a/src/cmd/new.rs +++ b/src/cmd/new.rs @@ -1,5 +1,5 @@ use crate::api::VolumeApi; -use crate::api::config::LocalReader; +use crate::api::config::{ConfigBody, LocalReader}; use crate::model::types::VolumeResult; use crate::{ api::WorkspaceApi, @@ -216,7 +216,26 @@ impl<'a> WorkspaceApi<'a> { format, } => { let body = value.to_string(format.clone())?; - (origin.to_string(), Some(body), None, Some(value.clone())) + let (cfg, base_body) = if value.bases.is_some() { + if let Ok(ConfigPath::File { path }) = ConfigPath::from_str(origin) { + let reader = LocalReader {}; + let (merged, individual_bases) = self + .config + .resolve_extends_chain(&reader, &path, value.clone(), 0) + .await?; + let bases_yaml = individual_bases + .iter() + .map(|(p, b)| b.to_string(*format).map(|yaml| format!("# {}\n{}", p, yaml))) + .collect::, _>>()? + .join("\n---\n"); + (Some(merged), if bases_yaml.is_empty() { None } else { Some(bases_yaml) }) + } else { + (Some(value.clone()), None) + } + } else { + (Some(value.clone()), None) + }; + (origin.to_string(), Some(body), base_body, cfg) } ConfigSource::Path { value: path } => match path { ConfigPath::File { path } => { @@ -227,14 +246,19 @@ impl<'a> WorkspaceApi<'a> { let cfg = RoozCfg::deserialize_config(&body, fmt)?; let (cfg, base_body) = match cfg { - Some(c) if c.extends.is_some() => { + Some(c) if c.bases.is_some() => { let reader = LocalReader {}; - let merged = self + let (merged, individual_bases) = self .config .resolve_extends_chain(&reader, path.as_str(), c, 0) .await?; - let base_body = merged.to_string(fmt)?; - (Some(merged), Some(base_body)) + let bases_yaml = individual_bases + .iter() + .map(|(path, b)| b.to_string(fmt).map(|yaml| format!("# {}\n{}", path, yaml))) + .collect::, _>>()? + .join("\n---\n"); + let base_body = if bases_yaml.is_empty() { None } else { Some(bases_yaml) }; + (Some(merged), base_body) } other => (other, None), }; @@ -247,30 +271,12 @@ impl<'a> WorkspaceApi<'a> { .await?; let (rooz_cfg, main_body, base_body) = match result { - Some((body, extends_body)) => { + Some(ConfigBody { body, bases, merged }) => { let fmt = FileFormat::from_path(&file_path); - let cfg = RoozCfg::deserialize_config(&body, fmt)?; - let merged = match cfg { - Some(c) if extends_body.is_some() => { - let eb = extends_body.as_deref().unwrap(); - let ext_path = c.extends.as_deref().unwrap(); - let ext_fmt = FileFormat::from_path(ext_path); - match RoozCfg::deserialize_config(eb, ext_fmt)? { - Some(mut base) => { - base.from_config(&c); - Some(base) - } - None => { - return Err( - "Failed to parse extends: invalid config" - .into(), - ); - } - } - } - other => other, - }; - (merged, Some(body), extends_body) + let cfg = merged + .map(Ok) + .unwrap_or_else(|| RoozCfg::deserialize_config(&body, fmt).map(|o| o.unwrap()))?; + (Some(cfg), Some(body), bases) } None => (None, None, None), }; @@ -283,9 +289,9 @@ impl<'a> WorkspaceApi<'a> { self.config .store(workspace_key, &origin, &body.unwrap()) .await?; - if let Some(eb) = extends_body { - self.config.store_extends(workspace_key, &eb).await?; - } + self.config + .store_bases(workspace_key, extends_body.as_deref().unwrap_or("")) + .await?; rooz_cfg } else { @@ -350,8 +356,8 @@ impl<'a> WorkspaceApi<'a> { (Some((body, _extends_body, format)), _) => { match RoozCfg::deserialize_config(body, *format)? { Some(c) => { - if c.extends.is_some() { - return Err("'extends' is not supported in in-repo config (.rooz.yaml); use it in a --config file instead".into()); + if c.bases.is_some() { + return Err("'bases' is not supported in in-repo config (.rooz.yaml); use it in a --config file instead".into()); } cfg_builder.from_config(&c); log::debug!("Config file applied."); diff --git a/src/cmd/update.rs b/src/cmd/update.rs index 036cb69..1dfe010 100644 --- a/src/cmd/update.rs +++ b/src/cmd/update.rs @@ -60,8 +60,8 @@ impl<'a> WorkspaceApi<'a> { .git .clone_config_repo(clone_env, &url, &file_path) .await?; - if let Some((body, _)) = result { - original_body = body; + if let Some(cb) = result { + original_body = cb.body; }; } }; diff --git a/src/config/config.rs b/src/config/config.rs index 57f7e8d..d999e3b 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -65,7 +65,7 @@ impl<'a> ConfigPath { #[derive(Debug, Clone, Copy)] pub enum ConfigType { Body, - Extends, + Bases, Runtime, } @@ -73,7 +73,7 @@ impl ConfigType { pub fn file_path(&self) -> &str { match self { ConfigType::Body => "workspace.config", - ConfigType::Extends => "extends.config", + ConfigType::Bases => "bases.config", ConfigType::Runtime => "runtime.config", } } @@ -159,7 +159,7 @@ impl RoozSidecar { #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(deny_unknown_fields)] pub struct RoozCfg { - pub extends: Option, + pub bases: Option>, pub vars: Option>, pub secrets: Option>, pub git_ssh_url: Option, @@ -183,7 +183,7 @@ pub struct RoozCfg { impl Default for RoozCfg { fn default() -> Self { Self { - extends: None, + bases: None, vars: Some(IndexMap::new()), secrets: Some(IndexMap::new()), git_ssh_url: None, @@ -219,17 +219,27 @@ impl RoozCfg { }) } - fn extend_if_any + IntoIterator>( - target: Option, - other: Option, - ) -> Option { - match (target, other) { - (Some(mut t), Some(o)) => { - t.extend(o); - Some(t) - } - (t, None) => t, - (None, o) => o, + pub fn none() -> Self { + Self { + bases: None, + vars: None, + secrets: None, + git_ssh_url: None, + extra_repos: None, + image: None, + caches: None, + shell: None, + user: None, + ports: None, + privileged: None, + init: None, + install: None, + command: None, + args: None, + env: None, + sidecars: None, + data: None, + mounts: None, } } @@ -240,45 +250,59 @@ impl RoozCfg { user: cli.user.clone().or(self.user.clone()), git_ssh_url: cli.git_ssh_url.clone().or(self.git_ssh_url.clone()), privileged: cli.privileged.or(self.privileged), - caches: Self::extend_if_any(self.caches.clone(), cli.caches.clone()), + caches: extend_if_any(self.caches.clone(), cli.caches.clone()), ..self.clone() } } pub fn from_config(&mut self, config: &RoozCfg) -> () { *self = RoozCfg { - extends: None, - vars: Self::extend_if_any(self.vars.clone(), config.vars.clone()), - secrets: Self::extend_if_any(self.secrets.clone(), config.secrets.clone()), + bases: None, + vars: extend_if_any(self.vars.clone(), config.vars.clone()), + secrets: extend_if_any(self.secrets.clone(), config.secrets.clone()), git_ssh_url: config.git_ssh_url.clone().or(self.git_ssh_url.clone()), - extra_repos: Self::extend_if_any(self.extra_repos.clone(), config.extra_repos.clone()), + extra_repos: extend_if_any(self.extra_repos.clone(), config.extra_repos.clone()), image: config.image.clone().or(self.image.clone()), - caches: Self::extend_if_any(self.caches.clone(), config.caches.clone()), + caches: extend_if_any(self.caches.clone(), config.caches.clone()), shell: config.shell.clone().or(self.shell.clone()), user: config.user.clone().or(self.user.clone()), - ports: Self::extend_if_any(self.ports.clone(), config.ports.clone()), + ports: extend_if_any(self.ports.clone(), config.ports.clone()), privileged: config.privileged.clone().or(self.privileged.clone()), init: config.init.clone().or(self.init.clone()), - command: Self::extend_if_any(self.command.clone(), config.command.clone()), - args: Self::extend_if_any(self.args.clone(), config.args.clone()), - env: Self::extend_if_any(self.env.clone(), config.env.clone()), - sidecars: Self::extend_if_any(self.sidecars.clone(), config.sidecars.clone()), - data: Self::extend_if_any(self.data.clone(), config.data.clone()), - mounts: Self::extend_if_any(self.mounts.clone(), config.mounts.clone()), + command: config.command.clone().or(self.command.clone()), + args: config.args.clone().or(self.args.clone()), + env: extend_if_any(self.env.clone(), config.env.clone()), + sidecars: merge_sidecars(self.sidecars.clone(), config.sidecars.clone()), + data: extend_if_any(self.data.clone(), config.data.clone()), + mounts: extend_if_any(self.mounts.clone(), config.mounts.clone()), install: config.install.clone().or(self.install.clone()), } } - pub fn validate_extends_path(path: &str) -> Result<(), AnyError> { + pub fn validate_base_path(path: &str) -> Result<(), AnyError> { if path.contains(':') { return Err(format!( - "extends path must be a local relative path (no URLs): '{}'", + "base path must be a local relative path (no URLs): '{}'", path ) .into()); } if path.starts_with('/') { - return Err(format!("extends path must be relative, not absolute: '{}'", path).into()); + return Err(format!("base path must be relative, not absolute: '{}'", path).into()); + } + Ok(()) + } + + pub fn validate_base_list(paths: &[String]) -> Result<(), AnyError> { + if paths.len() > 2 { + return Err(format!( + "at most 2 base paths allowed per level, got {}", + paths.len() + ) + .into()); + } + for path in paths { + Self::validate_base_path(path)?; } Ok(()) } @@ -288,7 +312,7 @@ impl RoozCfg { shell: cli.env.shell.map(|v| vec![v]).or(self.shell.clone()), image: cli.env.image.or(self.image.clone()), user: cli.env.user.or(self.user.clone()), - caches: Self::extend_if_any(self.caches.clone(), cli.env.caches), + caches: extend_if_any(self.caches.clone(), cli.env.caches), git_ssh_url: cli.git_ssh_url.or(self.git_ssh_url.clone()), ..self.clone() } @@ -416,6 +440,60 @@ impl RoozCfg { } } } +fn extend_if_any + IntoIterator>( + target: Option, + other: Option, +) -> Option { + match (target, other) { + (Some(mut t), Some(o)) => { + t.extend(o); + Some(t) + } + (t, None) => t, + (None, o) => o, + } +} + +fn merge_sidecars( + target: Option>, + other: Option>, +) -> Option> { + match (target, other) { + (Some(mut t), Some(o)) => { + for (k, v) in o { + match t.get_mut(&k) { + Some(existing) => existing.merge_from(&v), + None => { + t.insert(k, v); + } + } + } + Some(t) + } + (t, None) => t, + (None, o) => o, + } +} + +impl RoozSidecar { + pub fn merge_from(&mut self, other: &RoozSidecar) { + self.image = other.image.clone(); + self.env = extend_if_any(self.env.clone(), other.env.clone()); + self.command = other.command.clone().or(self.command.clone()); + self.args = other.args.clone().or(self.args.clone()); + self.shell = other.shell.clone().or(self.shell.clone()); + self.mounts = extend_if_any(self.mounts.clone(), other.mounts.clone()); + self.ports = extend_if_any(self.ports.clone(), other.ports.clone()); + self.privileged = other.privileged.or(self.privileged); + self.init = other.init.or(self.init); + self.install = other.install.clone().or(self.install.clone()); + self.work_dir = other.work_dir.clone().or(self.work_dir.clone()); + self.user = other.user.clone().or(self.user.clone()); + self.uid = other.uid.or(self.uid); + self.egress = other.egress.or(self.egress); + } +} + fn render_str( reg: &Handlebars, val: &str, diff --git a/src/util/git.rs b/src/util/git.rs index 59083d8..6e1911f 100644 --- a/src/util/git.rs +++ b/src/util/git.rs @@ -2,7 +2,7 @@ use gix_config::File; use std::collections::HashMap; use crate::{ - api::{GitApi, container}, + api::{GitApi, config::ConfigBody, container}, config::config::FileFormat, constants, model::{ @@ -246,7 +246,7 @@ impl<'a> GitApi<'a> { spec: CloneEnv, url: &str, path: &str, - ) -> Result<(Option<(String, Option)>, String), AnyError> { + ) -> Result<(Option, String), AnyError> { let container_id = self .clone_from_spec( &CloneEnv {