Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
name: CI

on:
push:
branches:
- main
pull_request:

env:
CARGO_TERM_COLOR: always

jobs:
validate:
name: Validate
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy, llvm-tools-preview

- name: Cache Cargo
uses: Swatinem/rust-cache@v2

- name: Install cargo-deny
uses: taiki-e/install-action@v2
with:
tool: cargo-deny

- name: Install cargo-llvm-cov
uses: taiki-e/install-action@v2
with:
tool: cargo-llvm-cov

- name: Check formatting
run: cargo fmt --check

- name: Clippy
run: cargo clippy --all-targets -- -D warnings

- name: Test
run: cargo test --all-targets

- name: Supply-chain checks
run: cargo deny check

- name: Coverage
run: cargo llvm-cov --all-targets --fail-under-lines 80 --summary-only
6 changes: 3 additions & 3 deletions src/backend/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ pub fn from_config(cfg: &BigstoreConfig) -> Result<Backend> {
let s = store::build_object_store(&cfg.backend)?;
Ok(Backend::ObjectStore(Arc::from(s)))
}
BackendConfig::Rclone { remote } => Ok(Backend::Rclone(rclone::RcloneBackend::new(
remote.clone(),
))),
BackendConfig::Rclone { remote } => {
Ok(Backend::Rclone(rclone::RcloneBackend::new(remote.clone())))
}
BackendConfig::Local { path } => {
let s = store::build_local_store(path)?;
Ok(Backend::ObjectStore(Arc::from(s)))
Expand Down
35 changes: 17 additions & 18 deletions src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,10 @@ pub enum BackendConfig {
},

#[serde(rename = "rclone")]
Rclone {
remote: String,
},
Rclone { remote: String },

#[serde(rename = "local")]
Local {
path: String,
},
Local { path: String },
}

impl BigstoreConfig {
Expand Down Expand Up @@ -124,8 +120,7 @@ impl BigstoreConfig {
let content = std::fs::read_to_string(path)
.with_context(|| format!("failed to read {}", path.display()))?;
// Layout is validated during deserialization — invalid templates fail here
toml::from_str(&content)
.with_context(|| format!("failed to parse {}", path.display()))
toml::from_str(&content).with_context(|| format!("failed to parse {}", path.display()))
}

/// Find and load config from a repo root.
Expand All @@ -137,12 +132,12 @@ impl BigstoreConfig {
}
let legacy_path = repo_root.join(".bigstore");
if legacy_path.exists() {
eprintln!("note: using legacy .bigstore config; run `git bigstore migrate-config` to upgrade");
eprintln!(
"note: using legacy .bigstore config; run `git bigstore migrate-config` to upgrade"
);
return Self::load(&legacy_path);
}
anyhow::bail!(
"no bigstore config found (looked for .bigstore.toml and .bigstore)"
)
anyhow::bail!("no bigstore config found (looked for .bigstore.toml and .bigstore)")
}

pub fn bucket_prefix(&self) -> &str {
Expand All @@ -167,7 +162,11 @@ impl BigstoreConfig {
/// Build the remote object key using the configured layout.
/// Safe: Layout is validated, Hexdigest is validated.
/// Returns Err if the layout doesn't support the given hash function.
pub fn remote_object_key(&self, hexdigest: &Hexdigest, hash_fn: HashFunction) -> Result<String> {
pub fn remote_object_key(
&self,
hexdigest: &Hexdigest,
hash_fn: HashFunction,
) -> Result<String> {
let key = self.layout.object_key(hexdigest, hash_fn)?;

let bucket_prefix = match &self.backend {
Expand Down Expand Up @@ -224,10 +223,7 @@ mod tests {
let cfg = BigstoreConfig::from_url("t3://my-bucket", None).unwrap();
match &cfg.backend {
BackendConfig::S3 { endpoint, .. } => {
assert_eq!(
endpoint.as_deref(),
Some("https://fly.storage.tigris.dev")
);
assert_eq!(endpoint.as_deref(), Some("https://fly.storage.tigris.dev"));
}
_ => panic!("expected S3"),
}
Expand All @@ -238,7 +234,10 @@ mod tests {
let cfg = BigstoreConfig::from_url("s3://bucket/data", None).unwrap();
let d = test_digest();
let key = cfg.remote_object_key(&d, HashFunction::Sha256).unwrap();
assert_eq!(key, format!("data/files/sha256/{}/{}", d.prefix(), d.rest()));
assert_eq!(
key,
format!("data/files/sha256/{}/{}", d.prefix(), d.rest())
);
}

#[test]
Expand Down
18 changes: 3 additions & 15 deletions src/dvc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,35 +286,23 @@ mod tests {
fn parse_dir_manifest_rejects_empty_relpath() {
let tmp = tempfile::NamedTempFile::new().unwrap();
let md5 = "aa".repeat(16);
std::fs::write(
tmp.path(),
format!(r#"[{{"md5":"{md5}","relpath":""}}]"#),
)
.unwrap();
std::fs::write(tmp.path(), format!(r#"[{{"md5":"{md5}","relpath":""}}]"#)).unwrap();
assert!(parse_dir_manifest(tmp.path()).is_err());
}

#[test]
fn parse_dir_manifest_rejects_dot_relpath() {
let tmp = tempfile::NamedTempFile::new().unwrap();
let md5 = "aa".repeat(16);
std::fs::write(
tmp.path(),
format!(r#"[{{"md5":"{md5}","relpath":"."}}]"#),
)
.unwrap();
std::fs::write(tmp.path(), format!(r#"[{{"md5":"{md5}","relpath":"."}}]"#)).unwrap();
let err = parse_dir_manifest(tmp.path()).unwrap_err();
assert!(err.to_string().contains("."), "{err}");
}

#[test]
fn parse_dir_manifest_rejects_bad_md5() {
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(
tmp.path(),
r#"[{"md5":"not-valid","relpath":"file.bin"}]"#,
)
.unwrap();
std::fs::write(tmp.path(), r#"[{"md5":"not-valid","relpath":"file.bin"}]"#).unwrap();
assert!(parse_dir_manifest(tmp.path()).is_err());
}
}
4 changes: 2 additions & 2 deletions src/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ pub fn clean() -> Result<()> {
tmp.flush()?;

let hex_str = hex::encode(hasher.finalize());
let hexdigest = Hexdigest::new(&hex_str, hash_fn)
.context("internal error: sha256 produced invalid hex")?;
let hexdigest =
Hexdigest::new(&hex_str, hash_fn).context("internal error: sha256 produced invalid hex")?;

let dest = cache::object_path(&git_dir, &hexdigest, hash_fn);
if let Some(parent) = dest.parent() {
Expand Down
32 changes: 20 additions & 12 deletions src/git.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,18 @@ impl FilterConfig {
}

// Partial presence
let clean = clean.ok_or_else(|| anyhow::anyhow!(
"filter.bigstore.smudge is set but filter.bigstore.clean is missing.\n\
let clean = clean.ok_or_else(|| {
anyhow::anyhow!(
"filter.bigstore.smudge is set but filter.bigstore.clean is missing.\n\
Fix: git config filter.bigstore.clean \"git-bigstore filter-clean\""
))?;
let smudge = smudge.ok_or_else(|| anyhow::anyhow!(
"filter.bigstore.clean is set but filter.bigstore.smudge is missing.\n\
)
})?;
let smudge = smudge.ok_or_else(|| {
anyhow::anyhow!(
"filter.bigstore.clean is set but filter.bigstore.smudge is missing.\n\
Fix: git config filter.bigstore.smudge \"git-bigstore filter-smudge\""
))?;
)
})?;

// Required must be "true"
match required.as_deref() {
Expand All @@ -111,16 +115,20 @@ impl FilterConfig {
}

// Command shape: must end with "filter-clean" / "filter-smudge"
let clean_bin = clean.strip_suffix(" filter-clean").ok_or_else(|| anyhow::anyhow!(
"filter.bigstore.clean has unexpected format: {clean:?}\n\
let clean_bin = clean.strip_suffix(" filter-clean").ok_or_else(|| {
anyhow::anyhow!(
"filter.bigstore.clean has unexpected format: {clean:?}\n\
Expected: \"<binary> filter-clean\"\n\
Fix: git config filter.bigstore.clean \"git-bigstore filter-clean\""
))?;
let smudge_bin = smudge.strip_suffix(" filter-smudge").ok_or_else(|| anyhow::anyhow!(
"filter.bigstore.smudge has unexpected format: {smudge:?}\n\
)
})?;
let smudge_bin = smudge.strip_suffix(" filter-smudge").ok_or_else(|| {
anyhow::anyhow!(
"filter.bigstore.smudge has unexpected format: {smudge:?}\n\
Expected: \"<binary> filter-smudge\"\n\
Fix: git config filter.bigstore.smudge \"git-bigstore filter-smudge\""
))?;
)
})?;

// Same binary prefix
anyhow::ensure!(
Expand Down
6 changes: 3 additions & 3 deletions src/lfs_adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
//! 1. .bigstore.toml (if present)
//! 2. git config bigstore-lfs.url (fallback for LFS-only repos)

use anyhow::{Context, Result};
use crate::{backend, config, git, transfer, types};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::io::{BufRead, BufReader, Write};
use std::path::Path;
Expand Down Expand Up @@ -326,8 +326,8 @@ pub fn run() -> Result<()> {
continue;
}

let event: Event =
serde_json::from_str(&line).with_context(|| format!("invalid JSON from LFS: {line}"))?;
let event: Event = serde_json::from_str(&line)
.with_context(|| format!("invalid JSON from LFS: {line}"))?;

match event.event.as_str() {
"init" => match load_config() {
Expand Down
Loading
Loading