diff --git a/.gitignore b/.gitignore index 26a8c694..25959e87 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ buck-out/ .vscode/ +src/.DS_Store +.DS_Store diff --git a/BUCK b/BUCK index d25eb0e4..959c3dd9 100644 --- a/BUCK +++ b/BUCK @@ -73,5 +73,6 @@ rust_library( "//third-party/rust/crates/tracing/0.1.41:tracing", "//third-party/rust/crates/uuid/1.18.1:uuid", "//third-party/rust/crates/zstd-sys/2.0.16+zstd.1.5.7:zstd-sys", + "//third-party/rust/crates/sha2/0.10.9:sha2", ], ) diff --git a/Cargo.lock b/Cargo.lock index bf5c745a..86e63181 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1290,6 +1290,7 @@ dependencies = [ "serde", "serde_json", "sha1", + "sha2", "similar", "tempfile", "thiserror 2.0.17", diff --git a/Cargo.toml b/Cargo.toml index a3f7bd3c..6cb58b9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ natord = "1.0.9" tempfile = "3.23.0" path-absolutize = "3.1.1" similar = "2.7.0" +sha2 = "0.10.9" [dev-dependencies] diff --git a/src/diff.rs b/src/diff.rs index 5b542ad7..18189c4e 100644 --- a/src/diff.rs +++ b/src/diff.rs @@ -1,4 +1,4 @@ -use crate::hash::SHA1; +use crate::hash::ObjectHash; use path_absolutize::Absolutize; use similar::{Algorithm, ChangeTag, TextDiff}; use std::collections::{HashMap, HashSet, VecDeque}; @@ -87,13 +87,13 @@ impl Diff { /// Compute diffs for a set of files, honoring an optional filter and emitting unified diffs. pub fn diff( - old_blobs: Vec<(PathBuf, SHA1)>, - new_blobs: Vec<(PathBuf, SHA1)>, + old_blobs: Vec<(PathBuf, ObjectHash)>, + new_blobs: Vec<(PathBuf, ObjectHash)>, filter: Vec, read_content: F, ) -> Vec where - F: Fn(&PathBuf, &SHA1) -> Vec, + F: Fn(&PathBuf, &ObjectHash) -> Vec, { let (processed_files, old_blobs_map, new_blobs_map) = Self::prepare_diff_data(old_blobs, new_blobs, &filter); @@ -148,13 +148,16 @@ impl Diff { /// Build maps, union file set, and apply filter/path checks fn prepare_diff_data( - old_blobs: Vec<(PathBuf, SHA1)>, - new_blobs: Vec<(PathBuf, SHA1)>, + old_blobs: Vec<(PathBuf, ObjectHash)>, + new_blobs: Vec<(PathBuf, ObjectHash)>, filter: &[PathBuf], - ) -> (Vec, HashMap, HashMap) { - let old_blobs_map: HashMap = old_blobs.into_iter().collect(); - let new_blobs_map: HashMap = new_blobs.into_iter().collect(); - + ) -> ( + Vec, + HashMap, + HashMap, + ) { + let old_blobs_map: HashMap = old_blobs.into_iter().collect(); + let new_blobs_map: HashMap = new_blobs.into_iter().collect(); // union set let union_files: HashSet = old_blobs_map .keys() @@ -174,8 +177,8 @@ impl Diff { fn should_process( file: &PathBuf, filter: &[PathBuf], - old_blobs: &HashMap, - new_blobs: &HashMap, + old_blobs: &HashMap, + new_blobs: &HashMap, ) -> bool { if !filter.is_empty() && !filter @@ -194,7 +197,7 @@ impl Diff { Ok(path_abs.starts_with(parent_abs)) } - fn short_hash(hash: Option<&SHA1>) -> String { + fn short_hash(hash: Option<&ObjectHash>) -> String { hash.map(|h| { let hex = h.to_string(); let take = Self::SHORT_HASH_LEN.min(hex.len()); @@ -206,9 +209,9 @@ impl Diff { /// Format a single file's unified diff string. pub fn diff_for_file_string( file: &PathBuf, - old_blobs: &HashMap, - new_blobs: &HashMap, - read_content: &dyn Fn(&PathBuf, &SHA1) -> Vec, + old_blobs: &HashMap, + new_blobs: &HashMap, + read_content: &dyn Fn(&PathBuf, &ObjectHash) -> Vec, ) -> String { let new_hash = new_blobs.get(file); let old_hash = old_blobs.get(file); @@ -221,8 +224,8 @@ impl Diff { /// Format a single file's unified diff using preloaded bytes to avoid re-reading. fn diff_for_file_preloaded( file: &Path, - old_hash: Option<&SHA1>, - new_hash: Option<&SHA1>, + old_hash: Option<&ObjectHash>, + new_hash: Option<&ObjectHash>, old_bytes: &[u8], new_bytes: &[u8], ) -> String { @@ -478,19 +481,23 @@ pub fn compute_diff(old_lines: &[String], new_lines: &[String]) -> Vec (String, SHA1, SHA1) { + fn run_diff( + logical_path: &str, + old_bytes: &[u8], + new_bytes: &[u8], + ) -> (String, ObjectHash, ObjectHash) { let file = PathBuf::from(logical_path); - let old_hash = SHA1::new(old_bytes); - let new_hash = SHA1::new(new_bytes); + let old_hash = ObjectHash::new(old_bytes); + let new_hash = ObjectHash::new(new_bytes); - let mut blob_store: HashMap> = HashMap::new(); + let mut blob_store: HashMap> = HashMap::new(); blob_store.insert(old_hash, old_bytes.to_vec()); blob_store.insert(new_hash, new_bytes.to_vec()); @@ -499,14 +506,15 @@ mod tests { old_map.insert(file.clone(), old_hash); new_map.insert(file.clone(), new_hash); - let reader = - |_: &PathBuf, h: &SHA1| -> Vec { blob_store.get(h).cloned().unwrap_or_default() }; + let reader = |_: &PathBuf, h: &ObjectHash| -> Vec { + blob_store.get(h).cloned().unwrap_or_default() + }; let diff = Diff::diff_for_file_string(&file, &old_map, &new_map, &reader); (diff, old_hash, new_hash) } - fn short_hash(hash: &SHA1) -> String { + fn short_hash(hash: &ObjectHash) -> String { hash.to_string().chars().take(7).collect() } @@ -514,8 +522,8 @@ mod tests { logical_path: &str, old_bytes: &[u8], new_bytes: &[u8], - old_hash: &SHA1, - new_hash: &SHA1, + old_hash: &ObjectHash, + new_hash: &ObjectHash, ) -> Option { let temp_dir = tempdir().ok()?; let old_file = temp_dir.path().join("old.txt"); @@ -564,6 +572,7 @@ mod tests { #[test] fn unified_diff_basic_changes() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old = b"a\nb\nc\n" as &[u8]; let new = b"a\nB\nc\nd\n" as &[u8]; let (diff, _, _) = run_diff("foo.txt", old, new); @@ -580,6 +589,7 @@ mod tests { #[test] fn binary_files_detection() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old_bytes = vec![0u8, 159, 146, 150]; let new_bytes = vec![0xFF, 0x00, 0x01]; let (diff, _, _) = run_diff("bin.dat", &old_bytes, &new_bytes); @@ -588,6 +598,7 @@ mod tests { #[test] fn diff_matches_git_for_fixture() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); //use it to test SHA1/SHA-256 diffs as well let base: PathBuf = [env!("CARGO_MANIFEST_DIR"), "tests", "diff"] .iter() .collect(); @@ -628,6 +639,7 @@ mod tests { #[test] fn diff_matches_git_for_large_change() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old_lines: Vec = (0..5_000).map(|i| format!("line {i}")).collect(); let mut new_lines = old_lines.clone(); for idx in [10, 499, 1_234, 3_210, 4_999] { @@ -670,6 +682,7 @@ mod tests { #[test] fn compute_diff_operations_basic_mapping() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old_lines = vec!["a".to_string(), "b".to_string(), "c".to_string()]; let new_lines = vec![ "a".to_string(), diff --git a/src/hash.rs b/src/hash.rs index 9d22ca89..944eeda8 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -3,30 +3,21 @@ //! location in the Git internal and mega database. //! -use std::{fmt::Display, io}; +use std::{cell::RefCell, fmt::Display, hash::Hash, io, str::FromStr}; +use crate::internal::object::types::ObjectType; use bincode::{Decode, Encode}; use colored::Colorize; use serde::{Deserialize, Serialize}; use sha1::Digest; - -use crate::internal::object::types::ObjectType; - +pub type SHA1 = ObjectHash; /// The [`SHA1`] struct, encapsulating a `[u8; 20]` array, is specifically designed to represent Git hash IDs. /// In Git's context, these IDs are 40-character hexadecimal strings generated via the SHA-1 algorithm. /// Each Git object receives a unique hash ID based on its content, serving as an identifier for its location /// within the Git internal database. Utilizing a dedicated struct for these hash IDs enhances code readability and /// maintainability by providing a clear, structured format for their manipulation and storage. /// -/// ### Change Log -/// -/// In previous versions of the 'mega' project, `Hash` was used to denote hash values. However, in newer versions, -/// `SHA1` is employed for this purpose. Future updates plan to extend support to SHA256 and SHA512, or potentially -/// other hash algorithms. By abstracting the hash model to `Hash`, and using specific imports like `use crate::hash::SHA1` -/// or `use crate::hash::SHA256`, the codebase maintains a high level of clarity and maintainability. This design choice -/// allows for easier adaptation to different hash algorithms while keeping the underlying implementation consistent and -/// understandable. - Nov 26, 2023 (by @genedna) -/// +/// The [`HashKind`] enum represents different types of hash algorithms supported in Git, #[derive( Clone, Copy, @@ -42,103 +33,205 @@ use crate::internal::object::types::ObjectType; Encode, Decode, )] -pub struct SHA1(pub [u8; 20]); +pub enum HashKind { + #[default] + Sha1, + Sha256, +} +/// Implementation of methods for the [`HashKind`] enum. +impl HashKind { + pub const fn size(&self) -> usize { + match self { + HashKind::Sha1 => 20, + HashKind::Sha256 => 32, + // Add more hash kinds here as needed + } + } + pub const fn hex_len(&self) -> usize { + match self { + HashKind::Sha1 => 40, + HashKind::Sha256 => 64, + } + } + pub const fn as_str(&self) -> &'static str { + match self { + HashKind::Sha1 => "sha1", + HashKind::Sha256 => "sha256", + } + } +} +impl std::fmt::Display for HashKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} +impl std::str::FromStr for HashKind { + type Err = String; -/// Display trait for SHA1. -impl Display for SHA1 { - /// Allows [`SHA1::to_string()`] to be used. - /// Note: If you want a terminal-friendly colorized output, use [`SHA1::to_color_str()`]. - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", hex::encode(self.0)) + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "sha1" => Ok(HashKind::Sha1), + "sha256" => Ok(HashKind::Sha256), + _ => Err("Invalid hash kind".to_string()), + } } } -impl AsRef<[u8]> for SHA1 { +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Deserialize, Serialize, Encode, Decode, +)] +pub enum ObjectHash { + Sha1([u8; 20]), + Sha256([u8; 32]), +} +impl Default for ObjectHash { + fn default() -> Self { + ObjectHash::Sha1([0u8; 20]) + } +} +impl Display for ObjectHash { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", hex::encode(self.as_ref())) + } +} +impl AsRef<[u8]> for ObjectHash { fn as_ref(&self) -> &[u8] { - &self.0 + match self { + ObjectHash::Sha1(bytes) => bytes.as_slice(), + ObjectHash::Sha256(bytes) => bytes.as_slice(), + } } } -/// Implementation of the [`std::str::FromStr`] trait for the [`SHA1`] type. -/// -/// To effectively use the `from_str` method for converting a string to a `SHA1` object, consider the following: -/// 1. The input string `s` should be a pre-calculated hexadecimal string, exactly 40 characters in length. This string -/// represents a SHA1 hash and should conform to the standard SHA1 hash format. +/// Implementation of the [`std::str::FromStr`] trait for the [`ObjectHash`] enum. +/// To effectively use the `from_str` method for converting a string to an `ObjectHash` object, consider the following: +/// 1. The input string `s` should be a pre-calculated hexadecimal string, either 40 characters in length for SHA1 or 64 characters for SHA256. +/// This string represents a hash and should conform to the standard hash format. /// 2. It is necessary to explicitly import the `FromStr` trait to utilize the `from_str` method. Include the import /// statement `use std::str::FromStr;` in your code before invoking the `from_str` function. This import ensures -/// that the `from_str` method is available for converting strings to `SHA1` objects. -impl std::str::FromStr for SHA1 { +impl FromStr for ObjectHash { type Err = String; fn from_str(s: &str) -> Result { - let mut h = SHA1::default(); - if s.len() != 40 { - return Err("The length of the string is not 40".to_string()); + match s.len() { + 40 => { + let mut h = [0u8; 20]; + let bytes = hex::decode(s).map_err(|e| e.to_string())?; + h.copy_from_slice(bytes.as_slice()); + Ok(ObjectHash::Sha1(h)) + } + 64 => { + let mut h = [0u8; 32]; + let bytes = hex::decode(s).map_err(|e| e.to_string())?; + h.copy_from_slice(bytes.as_slice()); + Ok(ObjectHash::Sha256(h)) + } + _ => Err("Invalid hash length".to_string()), } - let bytes = hex::decode(s).map_err(|e| e.to_string())?; - h.0.copy_from_slice(bytes.as_slice()); - Ok(h) } } -/// Implementation of the `SHA1` struct. -/// -/// The naming conventions for the methods in this implementation are designed to be intuitive and self-explanatory: -/// -/// 1. `new` Prefix: -/// Methods starting with `new` are used for computing an SHA-1 hash from given data, signifying the creation of -/// a new `SHA1` instance. For example, `pub fn new(data: &Vec) -> SHA1` takes a byte vector and calculates its SHA-1 hash. -/// -/// 2. `from` Prefix: -/// Methods beginning with `from` are intended for creating a `SHA1` instance from an existing, pre-calculated value. -/// This implies direct derivation of the `SHA1` object from the provided input. For instance, `pub fn from_bytes(bytes: &[u8]) -> SHA1` -/// constructs a `SHA1` from a 20-byte array representing an SHA-1 hash. +/// Implementation of methods for the [`ObjectHash`] enum. +/// 1. The `kind` method determines the type of hash (SHA1 or SHA256) based on the variant of the `ObjectHash` enum. +/// 2. The `size` method returns the size of the hash in bytes, utilizing the `kind` method to determine the appropriate size. +/// 3. The `new` method computes the hash of the provided data using the specified hash kind (SHA1 or SHA256) and returns +/// an `ObjectHash` instance containing the computed hash. +/// 4. `from` Prefix:Methods to create an `ObjectHash` from different sources: +/// - `from_type_and_data`: Constructs an `ObjectHash` from an object type and its associated data. +/// - `from_bytes`: Creates an `ObjectHash` from a byte slice, ensuring the length matches the expected hash size. +/// - `from_stream`: Reads bytes from a stream to create an `ObjectHash`, ensuring the correct number of bytes are read based on the hash kind. +/// 5. `to` Prefix:Methods to convert an `ObjectHash` to different formats: +/// - `to_color_str`: Converts the hash to a colored string representation for display purposes +/// - `to_data`: Converts the hash to a byte vector. +/// - `_to_string`: Converts the hash to a hexadecimal string representation. /// -/// 3. `to` Prefix: -/// Methods with the `to` prefix are used for outputting the `SHA1` value in various formats. This prefix indicates a transformation or -/// conversion of the `SHA1` instance into another representation. For example, `pub fn to_string(self) -> String` converts the SHA1 -/// value to a plain hexadecimal string, and `pub fn to_data(self) -> Vec` converts it into a byte vector. The `to` prefix -/// thus serves as a clear indicator that the method is exporting or transforming the SHA1 value into a different format. -/// -/// These method naming conventions (`new`, `from`, `to`) provide clarity and predictability in the API, making it easier for users -/// to understand the intended use and functionality of each method within the `SHA1` struct. -impl SHA1 { - // The size of the SHA-1 hash value in bytes - pub const SIZE: usize = 20; - - /// Calculate the SHA-1 hash of the byte slice, then create a Hash value - pub fn new(data: &[u8]) -> SHA1 { - let h = sha1::Sha1::digest(data); - SHA1::from_bytes(h.as_ref()) - } - /// Create a Hash from the object type and data - /// This function is used to create a SHA1 hash from the object type and data. - /// It constructs a byte vector that includes the object type, the size of the data, - /// and the data itself, and then computes the SHA1 hash of this byte vector. - /// - /// Hash compute <- {Object Type}+{ }+{Object Size（before compress）}+{\x00}+{Object Content(before compress)} - pub fn from_type_and_data(object_type: ObjectType, data: &[u8]) -> SHA1 { +impl ObjectHash { + /// returns a zeroed hash value for the given hash kind + pub fn zero_str(kind: HashKind) -> String { + match kind { + HashKind::Sha1 => "0000000000000000000000000000000000000000".to_string(), + HashKind::Sha256 => { + "0000000000000000000000000000000000000000000000000000000000000000".to_string() + } + } + } + + /// returns the kind of hash + pub fn kind(&self) -> HashKind { + match self { + ObjectHash::Sha1(_) => HashKind::Sha1, + ObjectHash::Sha256(_) => HashKind::Sha256, + } + } + /// returns the size of hash in bytes + pub fn size(&self) -> usize { + self.kind().size() + } + + /// Calculates the hash of the given data using the specified hash kind. + pub fn new(data: &[u8]) -> ObjectHash { + match get_hash_kind() { + HashKind::Sha1 => { + let h = sha1::Sha1::digest(data); + let mut bytes = [0u8; 20]; + bytes.copy_from_slice(h.as_ref()); + ObjectHash::Sha1(bytes) + } + HashKind::Sha256 => { + let h = sha2::Sha256::digest(data); + let mut bytes = [0u8; 32]; + bytes.copy_from_slice(h.as_ref()); + ObjectHash::Sha256(bytes) + } + } + } + /// Create ObjectHash from object type and data + pub fn from_type_and_data(object_type: ObjectType, data: &[u8]) -> ObjectHash { let mut d: Vec = Vec::new(); d.extend(object_type.to_data().unwrap()); d.push(b' '); d.extend(data.len().to_string().as_bytes()); d.push(b'\x00'); d.extend(data); - SHA1::new(&d) + ObjectHash::new(&d) } + /// Create ObjectHash from a byte slice + pub fn from_bytes(bytes: &[u8]) -> Result { + let expected_len = get_hash_kind().size(); + if bytes.len() != expected_len { + return Err(format!( + "Invalid byte length: got {}, expected {}", + bytes.len(), + expected_len + )); + } - /// Create Hash from a byte array, which is a 20-byte array already calculated - pub fn from_bytes(bytes: &[u8]) -> SHA1 { - let mut h = SHA1::default(); - h.0.copy_from_slice(bytes); - h + match get_hash_kind() { + HashKind::Sha1 => { + let mut h = [0u8; 20]; + h.copy_from_slice(bytes); + Ok(ObjectHash::Sha1(h)) + } + HashKind::Sha256 => { + let mut h = [0u8; 32]; + h.copy_from_slice(bytes); + Ok(ObjectHash::Sha256(h)) + } + } } - - /// Read the Hash value from the stream - /// This function will read exactly 20 bytes from the stream - pub fn from_stream(data: &mut impl io::Read) -> io::Result { - let mut h = SHA1::default(); - data.read_exact(&mut h.0)?; - Ok(h) + /// Create ObjectHash from a stream + pub fn from_stream(data: &mut impl io::Read) -> io::Result { + match get_hash_kind() { + HashKind::Sha1 => { + let mut h = [0u8; 20]; + data.read_exact(&mut h)?; + Ok(ObjectHash::Sha1(h)) + } + HashKind::Sha256 => { + let mut h = [0u8; 32]; + data.read_exact(&mut h)?; + Ok(ObjectHash::Sha256(h)) + } + } } /// Export sha1 value to String with the color @@ -148,16 +241,55 @@ impl SHA1 { /// Export sha1 value to a byte array pub fn to_data(self) -> Vec { - self.0.to_vec() + self.as_ref().to_vec() } /// [`core::fmt::Display`] is somewhat expensive, /// use this hack to get a string more efficiently pub fn _to_string(&self) -> String { - hex::encode(self.0) + hex::encode(self.as_ref()) } + + /// Get mutable hash as byte slice + pub fn as_mut_bytes(&mut self) -> &mut [u8] { + match self { + ObjectHash::Sha1(bytes) => bytes.as_mut_slice(), + ObjectHash::Sha256(bytes) => bytes.as_mut_slice(), + } + } +} +thread_local! { + /// Thread-local variable to store the current hash kind. + /// This allows different threads to work with different hash algorithms concurrently + /// without interfering with each other. + static CURRENT_HASH_KIND: RefCell = RefCell::new(HashKind::default()); +} +pub fn set_hash_kind(kind: HashKind) { + CURRENT_HASH_KIND.with(|h| { + *h.borrow_mut() = kind; + }); } +/// Retrieves the hash kind for the current thread. +pub fn get_hash_kind() -> HashKind { + CURRENT_HASH_KIND.with(|h| *h.borrow()) +} +/// A guard to reset the hash kind after the test +pub struct HashKindGuard { + prev: HashKind, +} +/// Implementation of the `Drop` trait for the `HashKindGuard` struct. +impl Drop for HashKindGuard { + fn drop(&mut self) { + set_hash_kind(self.prev); + } +} +/// Sets the hash kind for the current thread and returns a guard to reset it later. +pub fn set_hash_kind_for_test(kind: HashKind) -> HashKindGuard { + let prev = get_hash_kind(); + set_hash_kind(kind); + HashKindGuard { prev } +} #[cfg(test)] mod tests { @@ -168,24 +300,36 @@ mod tests { use std::str::FromStr; use std::{env, path::PathBuf}; - use crate::hash::SHA1; + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; #[test] fn test_sha1_new() { + // Set hash kind to SHA1 for this test + let _guard = set_hash_kind_for_test(HashKind::Sha1); // Example input let data = "Hello, world!".as_bytes(); // Generate SHA1 hash from the input data - let sha1 = SHA1::new(data); + let sha1 = ObjectHash::new(data); // Known SHA1 hash for "Hello, world!" let expected_sha1_hash = "943a702d06f34599aee1f8da8ef9f7296031d699"; assert_eq!(sha1.to_string(), expected_sha1_hash); } + #[test] + fn test_sha256_new() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let data = "Hello, world!".as_bytes(); + let sha256 = ObjectHash::new(data); + let expected_sha256_hash = + "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3"; + assert_eq!(sha256.to_string(), expected_sha256_hash); + } #[test] fn test_signature_without_delta() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/pack-1d0e6c14760c956c173ede71cb28f33d921e232f.pack"); @@ -195,63 +339,144 @@ mod tests { buffered.seek(SeekFrom::End(-20)).unwrap(); let mut buffer = vec![0; 20]; buffered.read_exact(&mut buffer).unwrap(); - let signature = SHA1::from_bytes(buffer.as_ref()); + let signature = ObjectHash::from_bytes(buffer.as_ref()).unwrap(); assert_eq!( signature.to_string(), "1d0e6c14760c956c173ede71cb28f33d921e232f" ); } + #[test] + fn test_signature_without_delta_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let f = std::fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + + buffered.seek(SeekFrom::End(-32)).unwrap(); + let mut buffer = vec![0; 32]; + buffered.read_exact(&mut buffer).unwrap(); + let signature = ObjectHash::from_bytes(buffer.as_ref()).unwrap(); + assert_eq!( + signature.to_string(), + "78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d" + ); + } #[test] fn test_sha1_from_bytes() { - let sha1 = SHA1::from_bytes(&[ + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let sha1 = ObjectHash::from_bytes(&[ 0x8a, 0xb6, 0x86, 0xea, 0xfe, 0xb1, 0xf4, 0x47, 0x02, 0x73, 0x8c, 0x8b, 0x0f, 0x24, 0xf2, 0x56, 0x7c, 0x36, 0xda, 0x6d, - ]); + ]) + .unwrap(); assert_eq!(sha1.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } + #[test] + fn test_sha256_from_bytes() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + // Pre-calculated SHA256 hash for "abc" + let sha256 = ObjectHash::from_bytes(&[ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, + 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, + 0xf2, 0x00, 0x15, 0xad, + ]) + .unwrap(); + + assert_eq!( + sha256.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } #[test] fn test_from_stream() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let source = [ 0x8a, 0xb6, 0x86, 0xea, 0xfe, 0xb1, 0xf4, 0x47, 0x02, 0x73, 0x8c, 0x8b, 0x0f, 0x24, 0xf2, 0x56, 0x7c, 0x36, 0xda, 0x6d, ]; let mut reader = std::io::Cursor::new(source); - let sha1 = SHA1::from_stream(&mut reader).unwrap(); + let sha1 = ObjectHash::from_stream(&mut reader).unwrap(); assert_eq!(sha1.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } - + #[test] + fn test_sha256_from_stream() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let source = [ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, + 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, + 0xf2, 0x00, 0x15, 0xad, + ]; + let mut reader = std::io::Cursor::new(source); + let sha256 = ObjectHash::from_stream(&mut reader).unwrap(); + assert_eq!( + sha256.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } #[test] fn test_sha1_from_str() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; - match SHA1::from_str(hash_str) { + match ObjectHash::from_str(hash_str) { Ok(hash) => { assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } Err(e) => println!("Error: {e}"), } } + #[test] + fn test_sha256_from_str() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash_str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + match ObjectHash::from_str(hash_str) { + Ok(hash) => { + assert_eq!( + hash.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + Err(e) => println!("Error: {e}"), + } + } #[test] fn test_sha1_to_string() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; - match SHA1::from_str(hash_str) { + match ObjectHash::from_str(hash_str) { Ok(hash) => { assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } Err(e) => println!("Error: {e}"), } } - + #[test] + fn test_sha256_to_string() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash_str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + match ObjectHash::from_str(hash_str) { + Ok(hash) => { + assert_eq!( + hash.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + Err(e) => println!("Error: {e}"), + } + } #[test] fn test_sha1_to_data() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; - match SHA1::from_str(hash_str) { + match ObjectHash::from_str(hash_str) { Ok(hash) => { assert_eq!( hash.to_data(), @@ -264,4 +489,22 @@ mod tests { Err(e) => println!("Error: {e}"), } } + #[test] + fn test_sha256_to_data() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash_str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + match ObjectHash::from_str(hash_str) { + Ok(hash) => { + assert_eq!( + hash.to_data(), + vec![ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, + 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad, + ] + ); + } + Err(e) => println!("Error: {e}"), + } + } } diff --git a/src/internal/index.rs b/src/internal/index.rs index 00b9c2e5..0ab41b19 100644 --- a/src/internal/index.rs +++ b/src/internal/index.rs @@ -9,12 +9,11 @@ use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; -use sha1::{Digest, Sha1}; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::{ObjectHash, get_hash_kind}; use crate::internal::pack::wrapper::Wrapper; -use crate::utils; +use crate::utils::{self, HashAlgorithm}; #[derive(PartialEq, Eq, Debug, Clone)] pub struct Time { @@ -112,7 +111,7 @@ pub struct IndexEntry { pub uid: u32, // 0 for windows pub gid: u32, // 0 for windows pub size: u32, - pub hash: SHA1, + pub hash: ObjectHash, pub flags: Flags, pub name: String, } @@ -138,7 +137,7 @@ impl Display for IndexEntry { impl IndexEntry { /** Metadata must be got by [fs::symlink_metadata] to avoid following symlink */ - pub fn new(meta: &fs::Metadata, hash: SHA1, name: String) -> Self { + pub fn new(meta: &fs::Metadata, hash: ObjectHash, name: String) -> Self { let mut entry = IndexEntry { ctime: Time::from_system_time(meta.created().unwrap()), mtime: Time::from_system_time(meta.modified().unwrap()), @@ -181,7 +180,7 @@ impl IndexEntry { /// - `file`: **to workdir path** /// - `workdir`: absolute or relative path - pub fn new_from_file(file: &Path, hash: SHA1, workdir: &Path) -> io::Result { + pub fn new_from_file(file: &Path, hash: ObjectHash, workdir: &Path) -> io::Result { let name = file.to_str().unwrap().to_string(); let file_abs = workdir.join(file); let meta = fs::symlink_metadata(file_abs)?; // without following symlink @@ -189,7 +188,7 @@ impl IndexEntry { Ok(index) } - pub fn new_from_blob(name: String, hash: SHA1, size: u32) -> Self { + pub fn new_from_blob(name: String, hash: ObjectHash, size: u32) -> Self { IndexEntry { ctime: Time { seconds: 0, @@ -266,7 +265,7 @@ impl Index { uid: file.read_u32::()?, gid: file.read_u32::()?, size: file.read_u32::()?, - hash: utils::read_sha1(file)?, + hash: utils::read_sha(file)?, flags: Flags::from(file.read_u16::()?), name: String::new(), }; @@ -281,14 +280,16 @@ impl Index { .insert((entry.name.clone(), entry.flags.stage), entry); // 1-8 nul bytes as necessary to pad the entry to a multiple of eight bytes - // while keeping the name NUL-terminated. // so at least 1 byte nul - let padding = 8 - ((22 + name_len) % 8); // 22 = sha1 + flags, others are 40 % 8 == 0 + // while keeping the name NUL-terminated. + let hash_len = get_hash_kind().size(); + let entry_len = hash_len + 2 + name_len; + let padding = 1 + ((8 - ((entry_len + 1) % 8)) % 8); // at least 1 byte nul utils::read_bytes(file, padding)?; } // Extensions - while file.bytes_read() + SHA1::SIZE < total_size as usize { - // The remaining 20 bytes must be checksum + while file.bytes_read() + get_hash_kind().size() < total_size as usize { + // The remaining bytes must be the pack checksum (size = get_hash_kind().size()) let sign = utils::read_bytes(file, 4)?; println!( "{:?}", @@ -310,7 +311,7 @@ impl Index { // check sum let file_hash = file.final_hash(); - let check_sum = utils::read_sha1(file)?; + let check_sum = utils::read_sha(file)?; if file_hash != check_sum { return Err(GitError::InvalidIndexFile("Check sum failed".to_string())); } @@ -320,7 +321,7 @@ impl Index { pub fn to_file(&self, path: impl AsRef) -> Result<(), GitError> { let mut file = File::create(path)?; - let mut hash = Sha1::new(); + let mut hash = HashAlgorithm::new(); let mut header = Vec::new(); header.write_all(b"DIRC")?; @@ -341,12 +342,13 @@ impl Index { entry_bytes.write_u32::(entry.uid)?; entry_bytes.write_u32::(entry.gid)?; entry_bytes.write_u32::(entry.size)?; - entry_bytes.write_all(&entry.hash.0)?; + entry_bytes.write_all(entry.hash.as_ref())?; entry_bytes.write_u16::((&entry.flags).try_into().unwrap())?; entry_bytes.write_all(entry.name.as_bytes())?; - let padding = 8 - ((22 + entry.name.len()) % 8); + let hash_len = get_hash_kind().size(); + let entry_len = hash_len + 2 + entry.name.len(); + let padding = 1 + ((8 - ((entry_len + 1) % 8)) % 8); // at least 1 byte nul entry_bytes.write_all(&vec![0; padding])?; - file.write_all(&entry_bytes)?; hash.update(&entry_bytes); } @@ -354,8 +356,9 @@ impl Index { // Extensions // check sum - let file_hash: [u8; 20] = hash.finalize().into(); - file.write_all(&file_hash)?; + let file_hash = + ObjectHash::from_bytes(&hash.finalize()).map_err(GitError::InvalidIndexFile)?; + file.write_all(file_hash.as_ref())?; Ok(()) } @@ -381,11 +384,11 @@ impl Index { )); let new_size = meta.len() as u32; - // re-calculate SHA1 + // re-calculate SHA1/SHA256 let mut file = File::open(&abs_path)?; - let mut hasher = Sha1::new(); + let mut hasher = HashAlgorithm::new(); io::copy(&mut file, &mut hasher)?; - let new_hash = SHA1::from_bytes(&hasher.finalize()); + let new_hash = ObjectHash::from_bytes(&hasher.finalize()).unwrap(); // refresh index if entry.ctime != new_ctime @@ -456,11 +459,11 @@ impl Index { self.entries.contains_key(&(name.to_string(), stage)) } - pub fn get_hash(&self, file: &str, stage: u8) -> Option { + pub fn get_hash(&self, file: &str, stage: u8) -> Option { self.get(file, stage).map(|entry| entry.hash) } - pub fn verify_hash(&self, file: &str, stage: u8, hash: &SHA1) -> bool { + pub fn verify_hash(&self, file: &str, stage: u8, hash: &ObjectHash) -> bool { let inner_hash = self.get_hash(file, stage); if let Some(inner_hash) = inner_hash { &inner_hash == hash @@ -541,7 +544,7 @@ impl Index { #[cfg(test)] mod tests { use super::*; - + use crate::hash::{HashKind, set_hash_kind_for_test}; #[test] fn test_time() { let time = Time { @@ -565,6 +568,7 @@ mod tests { #[test] fn test_index() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/index/index-760"); @@ -574,6 +578,18 @@ mod tests { println!("{entry}"); } } + #[test] + fn test_index_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/index/index-9-256"); + + let index = Index::from_file(source).unwrap(); + assert_eq!(index.size(), 9); + for (_, entry) in index.entries.iter() { + println!("{entry}"); + } + } #[test] fn test_index_to_file() { @@ -588,11 +604,24 @@ mod tests { #[test] fn test_index_entry_create() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("Cargo.toml"); let file = Path::new(source.as_path()); // use as a normal file - let hash = SHA1::from_bytes(&[0; 20]); + let hash = ObjectHash::from_bytes(&[0; 20]).unwrap(); + let workdir = Path::new("../"); + let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); + println!("{entry}"); + } + #[test] + fn test_index_entry_create_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("Cargo.toml"); + + let file = Path::new(source.as_path()); + let hash = ObjectHash::from_bytes(&[0u8; 32]).unwrap(); let workdir = Path::new("../"); let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); println!("{entry}"); diff --git a/src/internal/object/blob.rs b/src/internal/object/blob.rs index a2a2cc77..f846bcd0 100644 --- a/src/internal/object/blob.rs +++ b/src/internal/object/blob.rs @@ -17,7 +17,7 @@ //! $ echo "Hello, world!" | git hash-object -w --stdin //! ``` //! -//! This will output an SHA-1 hash, which is the ID of the newly created blob object. +//! This will output an SHA-1/ SHA-256 hash, which is the ID of the newly created blob object. //! The contents of the blob object would look something like this: //! //! ```bash @@ -30,14 +30,14 @@ use std::fmt::Display; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::types::ObjectType; /// **The Blob Object** #[derive(Eq, Debug, Clone)] pub struct Blob { - pub id: SHA1, + pub id: ObjectHash, pub data: Vec, } @@ -57,7 +57,7 @@ impl Display for Blob { impl ObjectTrait for Blob { /// Creates a new object from a byte slice. - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -94,8 +94,8 @@ impl Blob { /// - some file content can't be represented as a string (UTF-8), so we need to use bytes. pub fn from_content_bytes(content: Vec) -> Self { Blob { - // Calculate the SHA1 hash from the type and content - id: SHA1::from_type_and_data(ObjectType::Blob, &content), + // Calculate the hash from the type and content + id: ObjectHash::from_type_and_data(ObjectType::Blob, &content), data: content, } } @@ -104,8 +104,10 @@ impl Blob { #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; #[test] fn test_blob_from_content() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let content = "Hello, world!"; let blob = Blob::from_content(content); assert_eq!( @@ -113,4 +115,14 @@ mod tests { "5dd01c177f5d7d1be5346a5bc18a569a7410c2ef" ); } + #[test] + fn test_blob_from_content_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let content = "Hello, world!"; + let blob = Blob::from_content(content); + assert_eq!( + blob.id.to_string(), + "178b5fbed164aee269fee7323badf7269cca0eed0875717b0d2d4f9819164c3f" + ); + } } diff --git a/src/internal/object/commit.rs b/src/internal/object/commit.rs index 6112a001..40eb470a 100644 --- a/src/internal/object/commit.rs +++ b/src/internal/object/commit.rs @@ -5,7 +5,7 @@ //! //! Each commit object in Git contains the following information: //! -//! - A unique SHA-1 hash that identifies the commit. +//! - A unique SHA-1/ SHA-256 hash that identifies the commit. //! - The author and committer of the commit (which may be different people). //! - The date and time the commit was made. //! - A commit message that describes the changes made in the commit. @@ -15,7 +15,7 @@ use std::fmt::Display; use std::str::FromStr; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; use crate::internal::object::signature::Signature; @@ -36,9 +36,9 @@ use serde::Serialize; /// - The message field contains the commit message, which maybe include signed or DCO. #[derive(Eq, Debug, Clone, Serialize, Deserialize, Decode, Encode)] pub struct Commit { - pub id: SHA1, - pub tree_id: SHA1, - pub parent_commit_ids: Vec, + pub id: ObjectHash, + pub tree_id: ObjectHash, + pub parent_commit_ids: Vec, pub author: Signature, pub committer: Signature, pub message: String, @@ -65,12 +65,12 @@ impl Commit { pub fn new( author: Signature, committer: Signature, - tree_id: SHA1, - parent_commit_ids: Vec, + tree_id: ObjectHash, + parent_commit_ids: Vec, message: &str, ) -> Commit { let mut commit = Commit { - id: SHA1::default(), + id: ObjectHash::default(), tree_id, parent_commit_ids, author, @@ -79,7 +79,7 @@ impl Commit { }; // Calculate the hash of the commit object // The hash is calculated from the type and data of the commit object - let hash = SHA1::from_type_and_data(ObjectType::Commit, &commit.to_data().unwrap()); + let hash = ObjectHash::from_type_and_data(ObjectType::Commit, &commit.to_data().unwrap()); commit.id = hash; commit } @@ -89,13 +89,17 @@ impl Commit { /// and a fixed email address. /// It also sets the commit message to the provided string. /// # Arguments - /// - `tree_id`: The SHA1 hash of the tree object that this commit points to. - /// - `parent_commit_ids`: A vector of SHA1 hashes of the parent commits. + /// - `tree_id`: The SHA1/ SHA-256 hash of the tree object that this commit points to. + /// - `parent_commit_ids`: A vector of SHA1/ SHA-256 hashes of the parent commits. /// - `message`: A string containing the commit message. /// # Returns /// A new `Commit` object with the specified tree ID, parent commit IDs, and commit message. /// The author and committer signatures are generated using the current time and a fixed email address. - pub fn from_tree_id(tree_id: SHA1, parent_commit_ids: Vec, message: &str) -> Commit { + pub fn from_tree_id( + tree_id: ObjectHash, + parent_commit_ids: Vec, + message: &str, + ) -> Commit { let author = Signature::from_data( format!( "author mega {} +0800", @@ -149,14 +153,14 @@ impl Commit { } impl ObjectTrait for Commit { - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { let mut commit = data; // Find the tree id and remove it from the data let tree_end = commit.find_byte(0x0a).unwrap(); - let tree_id: SHA1 = SHA1::from_str( + let tree_id: ObjectHash = ObjectHash::from_str( String::from_utf8(commit[5..tree_end].to_owned()) // 5 is the length of "tree " .unwrap() .as_str(), @@ -170,12 +174,12 @@ impl ObjectTrait for Commit { // Find all parent commit ids // The parent commit ids are all the lines that start with "parent " // We can use find_iter to find all occurrences of "parent " - // and then extract the SHA1 hashes from them. - let parent_commit_ids: Vec = commit[..author_begin] + // and then extract the SHA1/ SHA-256 hashes from them. + let parent_commit_ids: Vec = commit[..author_begin] .find_iter("parent") .map(|parent| { let parent_end = commit[parent..].find_byte(0x0a).unwrap(); - SHA1::from_str( + ObjectHash::from_str( // 7 is the length of "parent " String::from_utf8(commit[parent + 7..parent + parent_end].to_owned()) .unwrap() @@ -249,9 +253,11 @@ impl ObjectTrait for Commit { #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; use std::str::FromStr; fn basic_commit() -> Commit { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let raw_commit = br#"tree 341e54913a3a43069f2927cc0f703e5a9f730df1 author benjamin.747 1757467768 +0800 committer benjamin.747 1757491219 +0800 @@ -276,22 +282,52 @@ gpgsig -----BEGIN PGP SIGNATURE----- test parse commit from bytes "#; - let hash = SHA1::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap(); + let hash = ObjectHash::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap(); Commit::from_bytes(raw_commit, hash).unwrap() } + fn basic_commit_sha256() -> Commit { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let raw_commit = br#"tree 0250024cf99636335fff1070e4220c5d8f67cb8633572d54b304629ad5382760 +parent 33324c6819589e8eed81d6c72f216469151a0f2dbe7f42ba021d8b63049eb754 +author jackieismpc 1764061895 +0800 +committer jackieismpc 1764061895 +0800 +gpgsig-sha256 -----BEGIN PGP SIGNATURE----- + + iQIzBAABCAAdFiEEzW/BI6wDXimDk/4lItD7G/h4TUsFAmklcscACgkQItD7G/h4 + TUtKFRAAtJq9tdl9XdND1ef2dXVQYCkQQlSdNHe2AR/QRVOPI39ZjD5aajRmZoE2 + rKDenNML1ruiGEm+K3ntRDjus+3QF5Xkhj1D6eImQt6RXyOlo64I+GLRKlzw80Sl + hrd+l1eeuS4n46Z0U9fo1Qgc/crSn2VhUtLHJjvRntJoOb1vNreI2Y42Zmal3oVT + fQNQ7mqzh3KuWoa8T6nVrLaLH1vl9qhRgkPcIRbFf+ECbB96qykHqcbdHuneSgfx + +REpr1cedilkQlX81JrQ8Ntf4QFUPPHALl27/G6oPLT714cflEbvcFw7rNR+ktcD + ZJIMu5Cl7X3/v5e0od/hF9uPfiLHckUsOXiMFLfqRdZx/5XeQFWRpq4eYcW7e89e + 3wJoBA2lCk8SHTBfsprKMpAweXJF9FCjRT5f9Zse2grqH81aQeNJnpSOoCq86oc/ + nxhi8+rbIbClLCGQoGF7sE/fvmKqcex++JnXHcHTtK002Gnh3oHX07sbahlcGuYY + kg4QhXiLTQ5GfXnEnTPdFqbOVG02vEEsNeRgkmOz4c8Pm1FTDyOkuXd/Igvy7A9R + MZwQcJ6E4MnsMnoH8FKswGqCD7ftwtJtRzryORBVzvPKALufIXDVLyBbae9dxdej + bcpUK1bGtDljlwNtbLIOu+F1y2OVh7Tn3zxaQLcEhbUe2tP6rGk= + =nJMO + -----END PGP SIGNATURE----- + +signed sha256 commit for test"#; + let hash = ObjectHash::from_str( + "ed43b50437e260a4d8fedacbaa38bad28b54cc424925e4180d9f186afaa0508c", + ) + .unwrap(); + Commit::from_bytes(raw_commit.as_bytes(), hash).unwrap() + } #[test] fn test_from_bytes_with_gpgsig() { let commit = basic_commit(); assert_eq!( commit.id, - SHA1::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap() + ObjectHash::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap() ); assert_eq!( commit.tree_id, - SHA1::from_str("341e54913a3a43069f2927cc0f703e5a9f730df1").unwrap() + ObjectHash::from_str("341e54913a3a43069f2927cc0f703e5a9f730df1").unwrap() ); assert_eq!(commit.author.name, "benjamin.747"); @@ -304,10 +340,41 @@ test parse commit from bytes assert!(commit.message.contains("-----END PGP SIGNATURE-----")); assert!(commit.message.contains("test parse commit from bytes")); } - + #[test] + fn test_from_bytes_with_gpgsig_sha256() { + let commit = basic_commit_sha256(); + assert_eq!( + commit.id, + ObjectHash::from_str( + "ed43b50437e260a4d8fedacbaa38bad28b54cc424925e4180d9f186afaa0508c" + ) + .unwrap() + ); + assert_eq!( + commit.tree_id, + ObjectHash::from_str( + "0250024cf99636335fff1070e4220c5d8f67cb8633572d54b304629ad5382760" + ) + .unwrap() + ); + assert_eq!(commit.author.name, "jackieismpc"); + assert_eq!(commit.author.email, "jackieismpc@gmail.com"); + assert_eq!(commit.committer.name, "jackieismpc"); + // // check message content (must contain gpgsig-sha256 and content) + assert!(commit.message.contains("-----BEGIN PGP SIGNATURE-----")); + assert!(commit.message.contains("-----END PGP SIGNATURE-----")); + assert!(commit.message.contains("signed sha256 commit for test")); + } #[test] fn test_format_message_with_pgp_signature() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let commit = basic_commit(); assert_eq!(commit.format_message(), "test parse commit from bytes"); } + #[test] + fn test_format_message_with_pgp_signature_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let commit = basic_commit_sha256(); + assert_eq!(commit.format_message(), "signed sha256 commit for test"); + } } diff --git a/src/internal/object/mod.rs b/src/internal/object/mod.rs index 1c1569a5..4478fa6f 100644 --- a/src/internal/object/mod.rs +++ b/src/internal/object/mod.rs @@ -17,11 +17,11 @@ use sha1::Digest; use crate::internal::object::types::ObjectType; use crate::internal::zlib::stream::inflate::ReadBoxed; -use crate::{errors::GitError, hash::SHA1}; +use crate::{errors::GitError, hash::ObjectHash}; pub trait ObjectTrait: Send + Sync + Display { /// Creates a new object from a byte slice. - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized; @@ -36,7 +36,11 @@ pub trait ObjectTrait: Send + Sync + Display { read.read_to_end(&mut content).unwrap(); let h = read.hash.clone(); let hash_str = h.finalize(); - Self::from_bytes(&content, SHA1::from_str(&format!("{hash_str:x}")).unwrap()).unwrap() + Self::from_bytes( + &content, + ObjectHash::from_str(&format!("{hash_str:x}")).unwrap(), + ) + .unwrap() } /// Returns the type of the object. diff --git a/src/internal/object/note.rs b/src/internal/object/note.rs index ab39e409..dd398eed 100644 --- a/src/internal/object/note.rs +++ b/src/internal/object/note.rs @@ -16,7 +16,7 @@ use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; @@ -27,10 +27,10 @@ use crate::internal::object::ObjectType; /// association managed through Git's reference system. #[derive(Eq, Debug, Clone, Serialize, Deserialize, Decode, Encode)] pub struct Note { - /// The SHA-1 hash of this Note object (same as the underlying Blob) - pub id: SHA1, - /// The SHA-1 hash of the object this Note annotates (usually a commit) - pub target_object_id: SHA1, + /// The ObjectHash of this Note object (same as the underlying Blob) + pub id: ObjectHash, + /// The ObjectHash of the object this Note annotates (usually a commit) + pub target_object_id: ObjectHash, /// The textual content of the Note pub content: String, } @@ -53,15 +53,15 @@ impl Note { /// Create a new Note for the specified target object with the given content /// /// # Arguments - /// * `target_object_id` - The SHA-1 hash of the object to annotate + /// * `target_object_id` - The ObjectHash of the object to annotate /// * `content` - The textual content of the note /// /// # Returns /// A new Note instance with calculated ID based on the content - pub fn new(target_object_id: SHA1, content: String) -> Self { - // Calculate the SHA-1 hash for this Note's content + pub fn new(target_object_id: ObjectHash, content: String) -> Self { + // Calculate the SHA-1/ SHA-256 hash for this Note's content // Notes are stored as Blob objects in Git - let id = SHA1::from_type_and_data(ObjectType::Blob, content.as_bytes()); + let id = ObjectHash::from_type_and_data(ObjectType::Blob, content.as_bytes()); Self { id, @@ -81,7 +81,7 @@ impl Note { /// # Returns /// A new Note instance with default target object ID pub fn from_content(content: &str) -> Self { - Self::new(SHA1::default(), content.to_string()) + Self::new(ObjectHash::default(), content.to_string()) } /// Get the size of the Note content in bytes @@ -100,8 +100,8 @@ impl Note { /// without changing the Note's content or ID. /// /// # Arguments - /// * `new_target` - The new target object SHA-1 hash - pub fn set_target(&mut self, new_target: SHA1) { + /// * `new_target` - The new target object SHA-1/ SHA-256 hash + pub fn set_target(&mut self, new_target: ObjectHash) { self.target_object_id = new_target; } @@ -112,15 +112,15 @@ impl Note { /// /// # Arguments /// * `data` - The raw byte data (UTF-8 encoded text content) - /// * `hash` - The SHA-1 hash of this Note object - /// * `target_object_id` - The SHA-1 hash of the object this Note annotates + /// * `hash` - The SHA-1/ SHA-256 hash of this Note object + /// * `target_object_id` - The SHA-1/ SHA-256 hash of the object this Note annotates /// /// # Returns /// A Result containing the Note object with complete association info pub fn from_bytes_with_target( data: &[u8], - hash: SHA1, - target_object_id: SHA1, + hash: ObjectHash, + target_object_id: ObjectHash, ) -> Result { let content = String::from_utf8(data.to_vec()) .map_err(|e| GitError::InvalidNoteObject(format!("Invalid UTF-8 content: {}", e)))?; @@ -139,7 +139,7 @@ impl Note { /// /// # Returns /// A tuple of (object_data, target_object_id) - pub fn to_data_with_target(&self) -> Result<(Vec, SHA1), GitError> { + pub fn to_data_with_target(&self) -> Result<(Vec, ObjectHash), GitError> { let data = self.to_data()?; Ok((data, self.target_object_id)) } @@ -150,11 +150,11 @@ impl ObjectTrait for Note { /// /// # Arguments /// * `data` - The raw byte data (UTF-8 encoded text content) - /// * `hash` - The SHA-1 hash of this Note object + /// * `hash` - The SHA-1/ SHA-256 hash of this Note object /// /// # Returns /// A Result containing the Note object or an error - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -164,7 +164,7 @@ impl ObjectTrait for Note { Ok(Note { id: hash, - target_object_id: SHA1::default(), // Target association managed externally + target_object_id: ObjectHash::default(), // Target association managed externally content, }) } @@ -193,17 +193,39 @@ impl ObjectTrait for Note { #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; use std::str::FromStr; #[test] fn test_note_creation_and_serialization() { - let target_id = SHA1::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let target_id = ObjectHash::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); let content = "This commit needs review".to_string(); let note = Note::new(target_id, content.clone()); assert_eq!(note.target_object_id, target_id); assert_eq!(note.content, content); - assert_ne!(note.id, SHA1::default()); + assert_ne!(note.id, ObjectHash::default()); + assert_eq!(note.get_type(), ObjectType::Blob); + + // Test serialization + let data = note.to_data().unwrap(); + assert_eq!(data, content.as_bytes()); + assert_eq!(note.get_size(), content.len()); + } + #[test] + fn test_note_creation_and_serialization_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let target_id = ObjectHash::from_str( + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ) + .unwrap(); + let content = "This commit needs review".to_string(); + let note = Note::new(target_id, content.clone()); + + assert_eq!(note.target_object_id, target_id); + assert_eq!(note.content, content); + assert_ne!(note.id, ObjectHash::default()); assert_eq!(note.get_type(), ObjectType::Blob); // Test serialization @@ -214,15 +236,42 @@ mod tests { #[test] fn test_note_deserialization() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let content = "Deserialization test content"; + let hash = ObjectHash::from_str("fedcba0987654321fedcba0987654321fedcba09").unwrap(); + let target_id = ObjectHash::from_str("abcdef1234567890abcdef1234567890abcdef12").unwrap(); + + // Test basic deserialization + let note = Note::from_bytes(content.as_bytes(), hash).unwrap(); + assert_eq!(note.content, content); + assert_eq!(note.id, hash); + assert_eq!(note.target_object_id, ObjectHash::default()); + + // Test deserialization with target + let note_with_target = + Note::from_bytes_with_target(content.as_bytes(), hash, target_id).unwrap(); + assert_eq!(note_with_target.content, content); + assert_eq!(note_with_target.id, hash); + assert_eq!(note_with_target.target_object_id, target_id); + } + #[test] + fn test_note_deserialization_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let content = "Deserialization test content"; - let hash = SHA1::from_str("fedcba0987654321fedcba0987654321fedcba09").unwrap(); - let target_id = SHA1::from_str("abcdef1234567890abcdef1234567890abcdef12").unwrap(); + let hash = ObjectHash::from_str( + "fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321", + ) + .unwrap(); + let target_id = ObjectHash::from_str( + "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + ) + .unwrap(); // Test basic deserialization let note = Note::from_bytes(content.as_bytes(), hash).unwrap(); assert_eq!(note.content, content); assert_eq!(note.id, hash); - assert_eq!(note.target_object_id, SHA1::default()); + assert_eq!(note.target_object_id, ObjectHash::default()); // Test deserialization with target let note_with_target = @@ -234,7 +283,29 @@ mod tests { #[test] fn test_note_with_target_methods() { - let target_id = SHA1::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let target_id = ObjectHash::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); + let content = "Test note with target methods"; + let note = Note::new(target_id, content.to_string()); + + // Test serialization with target + let (data, returned_target) = note.to_data_with_target().unwrap(); + assert_eq!(data, content.as_bytes()); + assert_eq!(returned_target, target_id); + + // Test deserialization with target + let restored_note = Note::from_bytes_with_target(&data, note.id, target_id).unwrap(); + assert_eq!(restored_note, note); + assert_eq!(restored_note.target_object_id, target_id); + assert_eq!(restored_note.content, content); + } + #[test] + fn test_note_with_target_methods_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let target_id = ObjectHash::from_str( + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ) + .unwrap(); let content = "Test note with target methods"; let note = Note::new(target_id, content.to_string()); @@ -252,26 +323,103 @@ mod tests { #[test] fn test_note_error_handling() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); // Test invalid UTF-8 let invalid_utf8 = vec![0xFF, 0xFE, 0xFD]; - let hash = SHA1::from_str("3333333333333333333333333333333333333333").unwrap(); - let target = SHA1::from_str("4444444444444444444444444444444444444444").unwrap(); - + let hash = ObjectHash::from_str("3333333333333333333333333333333333333333").unwrap(); + let target = ObjectHash::from_str("4444444444444444444444444444444444444444").unwrap(); let result = Note::from_bytes(&invalid_utf8, hash); assert!(result.is_err()); let result_with_target = Note::from_bytes_with_target(&invalid_utf8, hash, target); assert!(result_with_target.is_err()); } + #[test] + fn test_note_error_handling_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + // Test invalid UTF-8 + let invalid_utf8 = vec![0xFF, 0xFE, 0xFD]; + let hash = ObjectHash::from_str( + "3333333333333333333333333333333333333333333333333333333333333333", + ) + .unwrap(); + let target = ObjectHash::from_str( + "4444444444444444444444444444444444444444444444444444444444444444", + ) + .unwrap(); + let result = Note::from_bytes(&invalid_utf8, hash); + assert!(result.is_err()); + let result_with_target = Note::from_bytes_with_target(&invalid_utf8, hash, target); + assert!(result_with_target.is_err()); + } #[test] fn test_note_demo_functionality() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); + // This is a demonstration test that shows the complete functionality + // It's kept separate from unit tests for clarity + println!("\n🚀 Git Note Object Demo - Best Practices"); + println!("=========================================="); + + let commit_id = ObjectHash::from_str("a1b2c3d4e5f6789012345678901234567890abcd").unwrap(); + + println!("\n1️⃣ Creating a new Note object:"); + let note = Note::new( + commit_id, + "Code review: LGTM! Great implementation.".to_string(), + ); + println!(" Target Commit: {}", note.target_object_id); + println!(" Note ID: {}", note.id); + println!(" Content: {}", note.content); + println!(" Size: {} bytes", note.get_size()); + + println!("\n2️⃣ Serializing Note with target association:"); + let (serialized_data, target_id) = note.to_data_with_target().unwrap(); + println!(" Serialized size: {} bytes", serialized_data.len()); + println!(" Target object ID: {}", target_id); + println!( + " Git object format: blob {}\\0", + note.content.len() + ); + println!( + " Raw data preview: {:?}...", + &serialized_data[..std::cmp::min(30, serialized_data.len())] + ); + + println!("\n3️⃣ Basic deserialization (ObjectTrait):"); + let basic_note = Note::from_bytes(&serialized_data, note.id).unwrap(); + println!(" Successfully deserialized!"); + println!( + " Target Commit: {} (default - target managed externally)", + basic_note.target_object_id + ); + println!(" Content: {}", basic_note.content); + println!(" Content matches: {}", note.content == basic_note.content); + + println!("\n4️⃣ Best practice deserialization (with target):"); + let complete_note = + Note::from_bytes_with_target(&serialized_data, note.id, target_id).unwrap(); + println!(" Successfully deserialized with target!"); + println!(" Target Commit: {}", complete_note.target_object_id); + println!(" Content: {}", complete_note.content); + println!(" Complete objects are equal: {}", note == complete_note); + + // Basic assertions to ensure demo works + assert_eq!(note, complete_note); + assert_eq!(target_id, commit_id); + } + #[test] + fn test_note_demo_functionality_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); // This is a demonstration test that shows the complete functionality // It's kept separate from unit tests for clarity println!("\n🚀 Git Note Object Demo - Best Practices"); println!("=========================================="); - let commit_id = SHA1::from_str("a1b2c3d4e5f6789012345678901234567890abcd").unwrap(); + let commit_id = ObjectHash::from_str( + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ) + .unwrap(); println!("\n1️⃣ Creating a new Note object:"); let note = Note::new( diff --git a/src/internal/object/tag.rs b/src/internal/object/tag.rs index bd984281..986e3e49 100644 --- a/src/internal/object/tag.rs +++ b/src/internal/object/tag.rs @@ -42,7 +42,7 @@ use std::str::FromStr; use bstr::ByteSlice; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; use crate::internal::object::signature::Signature; @@ -50,8 +50,8 @@ use crate::internal::object::signature::Signature; /// The tag object is used to Annotated tag #[derive(Eq, Debug, Clone)] pub struct Tag { - pub id: SHA1, - pub object_hash: SHA1, + pub id: ObjectHash, + pub object_hash: ObjectHash, pub object_type: ObjectType, pub tag_name: String, pub tagger: Signature, @@ -85,7 +85,7 @@ impl Tag { // } pub fn new( - object_hash: SHA1, + object_hash: ObjectHash, object_type: ObjectType, tag_name: String, tagger: Signature, @@ -96,7 +96,7 @@ impl Tag { "object {}\ntype {}\ntag {}\ntagger {}\n\n{}", object_hash, object_type, tag_name, tagger, message ); - let id = SHA1::from_type_and_data(ObjectType::Tag, data.as_bytes()); + let id = ObjectHash::from_type_and_data(ObjectType::Tag, data.as_bytes()); Self { id, @@ -119,7 +119,7 @@ impl ObjectTrait for Tag { /// tagger 0x0a # The name, email address, and date of the person who created the annotated tag /// /// ``` - fn from_bytes(row_data: &[u8], hash: SHA1) -> Result + fn from_bytes(row_data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -131,7 +131,7 @@ impl ObjectTrait for Tag { headers = &headers[..pos]; } - let mut object_hash: Option = None; + let mut object_hash: Option = None; let mut object_type: Option = None; let mut tag_name: Option = None; let mut tagger: Option = None; @@ -141,7 +141,7 @@ impl ObjectTrait for Tag { let hash_str = s.to_str().map_err(|_| { GitError::InvalidTagObject("Invalid UTF-8 in object hash".to_string()) })?; - object_hash = Some(SHA1::from_str(hash_str).map_err(|_| { + object_hash = Some(ObjectHash::from_str(hash_str).map_err(|_| { GitError::InvalidTagObject("Invalid object hash format".to_string()) })?); } else if let Some(s) = line.strip_prefix(b"type ") { @@ -189,12 +189,13 @@ impl ObjectTrait for Tag { /// /// ```bash - /// object 0x0a # The SHA-1 hash of the object that the annotated tag is attached to (usually a commit) + /// object 0x0a # The SHA-1/ SHA-256 hash of the object that the annotated tag is attached to (usually a commit) /// type 0x0a #The type of Git object that the annotated tag is attached to (usually 'commit') /// tag 0x0a # The name of the annotated tag(in UTF-8 encoding) /// tagger 0x0a # The name, email address, and date of the person who created the annotated tag /// /// ``` + /// When using SHA-1, `` is 40 hex chars; when using SHA-256, it is 64 hex chars. fn to_data(&self) -> Result, GitError> { let mut data = Vec::new(); diff --git a/src/internal/object/tree.rs b/src/internal/object/tree.rs index 5bc7a74c..94d7a2f8 100644 --- a/src/internal/object/tree.rs +++ b/src/internal/object/tree.rs @@ -15,7 +15,7 @@ //! operations like merging and rebasing more quickly and accurately. //! use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::{ObjectHash, get_hash_kind}; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; use bincode::{Decode, Encode}; @@ -133,7 +133,7 @@ impl TreeItemMode { #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Hash, Encode, Decode)] pub struct TreeItem { pub mode: TreeItemMode, - pub id: SHA1, + pub id: ObjectHash, pub name: String, } @@ -151,7 +151,7 @@ impl Display for TreeItem { impl TreeItem { // Create a new TreeItem from a mode, id and name - pub fn new(mode: TreeItemMode, id: SHA1, name: String) -> Self { + pub fn new(mode: TreeItemMode, id: ObjectHash, name: String) -> Self { TreeItem { mode, id, name } } @@ -183,7 +183,7 @@ impl TreeItem { }; Ok(TreeItem { mode: TreeItemMode::tree_item_type_from_bytes(mode)?, - id: SHA1::from_bytes(id), + id: ObjectHash::from_bytes(id).unwrap(), name, }) } @@ -227,7 +227,7 @@ impl TreeItem { /// for each file or directory in the tree. #[derive(Eq, Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct Tree { - pub id: SHA1, + pub id: ObjectHash, pub tree_items: Vec, } @@ -262,7 +262,7 @@ impl Tree { } Ok(Tree { - id: SHA1::from_type_and_data(ObjectType::Tree, &data), + id: ObjectHash::from_type_and_data(ObjectType::Tree, &data), tree_items, }) } @@ -273,19 +273,19 @@ impl Tree { for item in &self.tree_items { data.extend_from_slice(item.to_data().as_slice()); } - self.id = SHA1::from_type_and_data(ObjectType::Tree, &data); + self.id = ObjectHash::from_type_and_data(ObjectType::Tree, &data); } } impl TryFrom<&[u8]> for Tree { type Error = GitError; fn try_from(data: &[u8]) -> Result { - let h = SHA1::from_type_and_data(ObjectType::Tree, data); + let h = ObjectHash::from_type_and_data(ObjectType::Tree, data); Tree::from_bytes(data, h) } } impl ObjectTrait for Tree { - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -295,8 +295,10 @@ impl ObjectTrait for Tree { // Find the position of the null byte (0x00) if let Some(index) = memchr::memchr(0x00, &data[i..]) { // Calculate the next position - let next = i + index + 21; - + let next = i + index + get_hash_kind().size() + 1; // +1 for the null byte + if next > data.len() { + return Err(GitError::InvalidTreeObject); + } //check bounds TreeItem::from_bytes will panic if out of bounds // Extract the bytes and create a TreeItem let item_data = &data[i..next]; let tree_item = TreeItem::from_bytes(item_data)?; @@ -339,16 +341,16 @@ impl ObjectTrait for Tree { #[cfg(test)] mod tests { - use std::str::FromStr; - - use crate::hash::SHA1; + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; use crate::internal::object::tree::{Tree, TreeItem, TreeItemMode}; + use std::str::FromStr; #[test] fn test_tree_item_new() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let tree_item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + ObjectHash::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), "hello-world".to_string(), ); @@ -358,12 +360,32 @@ mod tests { "8ab686eafeb1f44702738c8b0f24f2567c36da6d" ); } + #[test] + fn test_tree_item_new_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let tree_item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "hello-world".to_string(), + ); + + assert_eq!(tree_item.mode, TreeItemMode::Blob); + assert_eq!( + tree_item.id.to_string(), + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4" + ); + assert_eq!(tree_item.name, "hello-world"); + } #[test] fn test_tree_item_to_bytes() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let tree_item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + ObjectHash::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), "hello-world".to_string(), ); @@ -377,12 +399,46 @@ mod tests { ] ); } + #[test] + fn test_tree_item_to_bytes_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let tree_item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "hello-world".to_string(), + ); + + let bytes = tree_item.to_data(); + + // a small helper: convert hex string to byte sequence + fn hex_to_bytes(s: &str) -> Vec { + assert!(s.len() % 2 == 0); + let mut out = Vec::with_capacity(s.len() / 2); + for i in (0..s.len()).step_by(2) { + let byte = u8::from_str_radix(&s[i..i + 2], 16).unwrap(); + out.push(byte); + } + out + } + + // expected encoding: `"100644 hello-world\0" + <32-byte blob hash>` + let mut expected = b"100644 hello-world\0".to_vec(); + expected.extend_from_slice(&hex_to_bytes( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + )); + + assert_eq!(bytes, expected); + } #[test] fn test_tree_item_from_bytes() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + ObjectHash::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), "hello-world".to_string(), ); @@ -392,12 +448,30 @@ mod tests { assert_eq!(tree_item.mode, TreeItemMode::Blob); assert_eq!(tree_item.id.to_string(), item.id.to_string()); } + #[test] + fn test_tree_item_from_bytes_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "1111111111111111111111111111111111111111111111111111111111111111", + ) + .unwrap(), + "hello-world".to_string(), + ); + let bytes = item.to_data(); + let tree_item = TreeItem::from_bytes(bytes.as_slice()).unwrap(); + + assert_eq!(tree_item.mode, TreeItemMode::Blob); + assert_eq!(tree_item.id.to_string(), item.id.to_string()); + } #[test] fn test_from_tree_items() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("17288789afffb273c8c394bc65e87d899b92897b").unwrap(), + ObjectHash::from_str("17288789afffb273c8c394bc65e87d899b92897b").unwrap(), "hello-world".to_string(), ); let tree = Tree::from_tree_items(vec![item]).unwrap(); @@ -407,4 +481,56 @@ mod tests { tree.id.to_string() ); } + #[test] + fn test_from_tree_items_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let items = vec![ + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "a.txt".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "fc2593998f8e1dec9c3a8be11557888134dad90ef5c7a2d6236ed75534c7698e", + ) + .unwrap(), + "b.txt".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "21513dcb4d6f9eb247db3b4c52158395d94f809cbaa2630bd2a7a474d9b39fab", + ) + .unwrap(), + "c.txt".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "hello-world".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "9ba9ae56288652bf32f074f922e37d3e95df8920b3cdfc053309595b8f86cbc6", + ) + .unwrap(), + "message.txt".to_string(), + ), + ]; + let tree = Tree::from_tree_items(items).unwrap(); + println!("{}", tree.id); + assert_eq!( + "d712a36aadfb47cabc7aaa90cf9e515773ba3bfc1fe3783730b387ce15c49261", + tree.id.to_string() + ); + } } diff --git a/src/internal/pack/cache.rs b/src/internal/pack/cache.rs index fa9a0698..ed422af3 100644 --- a/src/internal/pack/cache.rs +++ b/src/internal/pack/cache.rs @@ -10,7 +10,7 @@ use dashmap::{DashMap, DashSet}; use lru_mem::LruCache; use threadpool::ThreadPool; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::pack::cache_object::{ ArcWrapper, CacheObject, FileLoadStore, MemSizeRecorder, }; @@ -20,29 +20,29 @@ pub trait _Cache { fn new(mem_size: Option, tmp_path: PathBuf, thread_num: usize) -> Self where Self: Sized; - fn get_hash(&self, offset: usize) -> Option; - fn insert(&self, offset: usize, hash: SHA1, obj: CacheObject) -> Arc; + fn get_hash(&self, offset: usize) -> Option; + fn insert(&self, offset: usize, hash: ObjectHash, obj: CacheObject) -> Arc; fn get_by_offset(&self, offset: usize) -> Option>; - fn get_by_hash(&self, h: SHA1) -> Option>; + fn get_by_hash(&self, h: ObjectHash) -> Option>; fn total_inserted(&self) -> usize; fn memory_used(&self) -> usize; fn clear(&self); } -impl lru_mem::HeapSize for SHA1 { +impl lru_mem::HeapSize for ObjectHash { fn heap_size(&self) -> usize { 0 } } pub struct Caches { - map_offset: DashMap, // offset to hash - hash_set: DashSet, // item in the cache + map_offset: DashMap, // offset to hash + hash_set: DashSet, // item in the cache // dropping large lru cache will take a long time on Windows without multi-thread IO // because "multi-thread IO" clone Arc, so it won't be dropped in the main thread, // and `CacheObjects` will be killed by OS after Process ends abnormally // Solution: use `mimalloc` - lru_cache: Mutex>>, + lru_cache: Mutex>>, mem_size: Option, tmp_path: PathBuf, path_prefixes: [Once; 256], @@ -52,14 +52,14 @@ pub struct Caches { impl Caches { /// only get object from memory, not from tmp file - fn try_get(&self, hash: SHA1) -> Option> { + fn try_get(&self, hash: ObjectHash) -> Option> { let mut map = self.lru_cache.lock().unwrap(); map.get(&hash).map(|x| x.data.clone()) } /// !IMPORTANT: because of the process of pack, the file must be written / be writing before, so it won't be dead lock /// fall back to temp to get item. **invoker should ensure the hash is in the cache, or it will block forever** - fn get_fallback(&self, hash: SHA1) -> io::Result> { + fn get_fallback(&self, hash: ObjectHash) -> io::Result> { let path = self.generate_temp_path(&self.tmp_path, hash); // read from tmp file let obj = { @@ -88,9 +88,10 @@ impl Caches { } /// generate the temp file path, hex string of the hash - fn generate_temp_path(&self, tmp_path: &Path, hash: SHA1) -> PathBuf { - // This is enough for the original path, 2 chars directory, 40 chars hash, and extra slashes - let mut path = PathBuf::with_capacity(self.tmp_path.capacity() + SHA1::SIZE * 2 + 5); + fn generate_temp_path(&self, tmp_path: &Path, hash: ObjectHash) -> PathBuf { + // Reserve capacity for base path, 2-char subdir, hex hash string, and separators + let mut path = + PathBuf::with_capacity(self.tmp_path.capacity() + hash.to_string().len() + 5); path.push(tmp_path); let hash_str = hash._to_string(); path.push(&hash_str[..2]); // use first 2 chars as the directory @@ -118,8 +119,9 @@ impl Caches { /// memory used by the index (exclude lru_cache which is contained in CacheObject::get_mem_size()) pub fn memory_used_index(&self) -> usize { - self.map_offset.capacity() * (std::mem::size_of::() + std::mem::size_of::()) - + self.hash_set.capacity() * (std::mem::size_of::()) + self.map_offset.capacity() + * (std::mem::size_of::() + std::mem::size_of::()) + + self.hash_set.capacity() * (std::mem::size_of::()) } /// remove the tmp dir @@ -156,11 +158,11 @@ impl _Cache for Caches { } } - fn get_hash(&self, offset: usize) -> Option { + fn get_hash(&self, offset: usize) -> Option { self.map_offset.get(&offset).map(|x| *x) } - fn insert(&self, offset: usize, hash: SHA1, obj: CacheObject) -> Arc { + fn insert(&self, offset: usize, hash: ObjectHash, obj: CacheObject) -> Arc { let obj_arc = Arc::new(obj); { // ? whether insert to cache directly or only write to tmp file @@ -189,7 +191,7 @@ impl _Cache for Caches { } } - fn get_by_hash(&self, hash: SHA1) -> Option> { + fn get_by_hash(&self, hash: ObjectHash) -> Option> { // check if the hash is in the cache( lru or tmp file) if self.hash_set.contains(&hash) { match self.try_get(hash) { @@ -235,12 +237,15 @@ mod test { use super::*; use crate::{ - hash::SHA1, + hash::{HashKind, ObjectHash, set_hash_kind_for_test}, internal::{object::types::ObjectType, pack::cache_object::CacheObjectInfo}, }; + use std::sync::Arc; + use std::thread; #[test] fn test_cache_single_thread() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); let tmp_path = source.clone().join("tests/.cache_tmp"); @@ -249,8 +254,8 @@ mod test { } let cache = Caches::new(Some(2048), tmp_path, 1); - let a_hash = SHA1::new(String::from("a").as_bytes()); - let b_hash = SHA1::new(String::from("b").as_bytes()); + let a_hash = ObjectHash::new(String::from("a").as_bytes()); + let b_hash = ObjectHash::new(String::from("b").as_bytes()); let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, a_hash), data_decompressed: vec![0; 800], @@ -276,7 +281,7 @@ mod test { assert!(cache.try_get(b_hash).is_some()); assert!(cache.try_get(a_hash).is_some()); - let c_hash = SHA1::new(String::from("c").as_bytes()); + let c_hash = ObjectHash::new(String::from("c").as_bytes()); // insert c which will evict both a and b let c = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, c_hash), @@ -291,4 +296,102 @@ mod test { assert!(cache.try_get(c_hash).is_some()); assert!(cache.get_by_hash(c_hash).is_some()); } + #[test] + fn test_cache_single_thread_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + let tmp_path = source.clone().join("tests/.cache_tmp_sha256"); + + if tmp_path.exists() { + fs::remove_dir_all(&tmp_path).unwrap(); + } + let cache = Caches::new(Some(4096), tmp_path, 1); + let a_hash = ObjectHash::new(String::from("a").as_bytes()); + let b_hash = ObjectHash::new(String::from("b").as_bytes()); + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, a_hash), + data_decompressed: vec![0; 1500], + mem_recorder: None, + offset: 0, + is_delta_in_pack: false, + }; + let b = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, b_hash), + data_decompressed: vec![0; 1500], + mem_recorder: None, + offset: 0, + is_delta_in_pack: false, + }; + // insert a + cache.insert(a.offset, a_hash, a.clone()); + assert!(cache.hash_set.contains(&a_hash)); + assert!(cache.try_get(a_hash).is_some()); + // insert b, a should still be in cache + cache.insert(b.offset, b_hash, b.clone()); + assert!(cache.hash_set.contains(&b_hash)); + assert!(cache.try_get(b_hash).is_some()); + assert!(cache.try_get(a_hash).is_some()); + let c_hash = ObjectHash::new(String::from("c").as_bytes()); + // insert c which will evict both a and b + let c = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, c_hash), + data_decompressed: vec![0; 3000], + mem_recorder: None, + offset: 0, + is_delta_in_pack: false, + }; + cache.insert(c.offset, c_hash, c.clone()); + assert!(cache.try_get(a_hash).is_none()); + assert!(cache.try_get(b_hash).is_none()); + assert!(cache.try_get(c_hash).is_some()); + assert!(cache.get_by_hash(c_hash).is_some()); + } + #[test] + /// consider the multi-threaded scenario where different threads use different hash kinds + fn test_cache_multi_thread_mixed_hash_kinds() { + let base = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + let tmp_path = base.join("tests/.cache_tmp_mixed"); + if tmp_path.exists() { + fs::remove_dir_all(&tmp_path).unwrap(); + } + + let cache = Arc::new(Caches::new(Some(4096), tmp_path, 2)); + + let cache_sha1 = Arc::clone(&cache); + let handle_sha1 = thread::spawn(move || { + let _g = set_hash_kind_for_test(HashKind::Sha1); + let hash = ObjectHash::new(b"sha1-entry"); + let obj = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash), + data_decompressed: vec![0; 800], + mem_recorder: None, + offset: 1, + is_delta_in_pack: false, + }; + cache_sha1.insert(obj.offset, hash, obj.clone()); + assert!(cache_sha1.hash_set.contains(&hash)); + assert!(cache_sha1.try_get(hash).is_some()); + }); + + let cache_sha256 = Arc::clone(&cache); + let handle_sha256 = thread::spawn(move || { + let _g = set_hash_kind_for_test(HashKind::Sha256); + let hash = ObjectHash::new(b"sha256-entry"); + let obj = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash), + data_decompressed: vec![0; 1500], + mem_recorder: None, + offset: 2, + is_delta_in_pack: false, + }; + cache_sha256.insert(obj.offset, hash, obj.clone()); + assert!(cache_sha256.hash_set.contains(&hash)); + assert!(cache_sha256.try_get(hash).is_some()); + }); + + handle_sha1.join().unwrap(); + handle_sha256.join().unwrap(); + + assert_eq!(cache.total_inserted(), 2); + } } diff --git a/src/internal/pack/cache_object.rs b/src/internal/pack/cache_object.rs index 146a3e80..52a22eb7 100644 --- a/src/internal/pack/cache_object.rs +++ b/src/internal/pack/cache_object.rs @@ -12,7 +12,7 @@ use threadpool::ThreadPool; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::pack::entry::Entry; use crate::internal::pack::utils; -use crate::{hash::SHA1, internal::object::types::ObjectType}; +use crate::{hash::ObjectHash, internal::object::types::ObjectType}; // /// record heap-size of all CacheObjects, used for memory limit. // static CACHE_OBJS_MEM_SIZE: AtomicUsize = AtomicUsize::new(0); @@ -56,16 +56,16 @@ impl Deserialize<'a>> FileLoadStore for T { pub(crate) enum CacheObjectInfo { /// The object is one of the four basic types: /// [`ObjectType::Blob`], [`ObjectType::Tree`], [`ObjectType::Commit`], or [`ObjectType::Tag`]. - /// The metadata contains the [`ObjectType`] and the [`SHA1`] hash of the object. - BaseObject(ObjectType, SHA1), + /// The metadata contains the [`ObjectType`] and the [`ObjectHash`] hash of the object. + BaseObject(ObjectType, ObjectHash), /// The object is an offset delta with a specified offset delta [`usize`], /// and the size of the expanded object (previously `delta_final_size`). OffsetDelta(usize, usize), /// Similar to [`OffsetDelta`], but delta algorithm is `zstd`. OffsetZstdelta(usize, usize), - /// The object is a hash delta with a specified [`SHA1`] hash, + /// The object is a hash delta with a specified [`ObjectHash`] hash, /// and the size of the expanded object (previously `delta_final_size`). - HashDelta(SHA1, usize), + HashDelta(ObjectHash, usize), } impl CacheObjectInfo { @@ -197,10 +197,10 @@ impl CacheObject { self.info.object_type() } - /// Get the [`SHA1`] hash of the object. + /// Get the [`ObjectHash`] hash of the object. /// /// If the object is a delta object, return [`None`]. - pub fn base_object_hash(&self) -> Option { + pub fn base_object_hash(&self) -> Option { match &self.info { CacheObjectInfo::BaseObject(_, hash) => Some(*hash), _ => None, @@ -220,7 +220,7 @@ impl CacheObject { /// Get the hash delta of the object. /// /// If the object is not a hash delta, return [`None`]. - pub fn hash_delta(&self) -> Option { + pub fn hash_delta(&self) -> Option { match &self.info { CacheObjectInfo::HashDelta(hash, _) => Some(*hash), _ => None, @@ -370,11 +370,12 @@ mod test { use lru_mem::LruCache; use super::*; - + use crate::hash::{HashKind, set_hash_kind_for_test}; #[test] fn test_heap_size_record() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut obj = CacheObject { - info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, data_decompressed: vec![0; 1024], mem_recorder: None, @@ -388,11 +389,30 @@ mod test { drop(obj); assert_eq!(mem.load(Ordering::Relaxed), 0); } + #[test] + fn test_heap_size_record_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut obj = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + let mem = Arc::new(AtomicUsize::default()); + assert_eq!(mem.load(Ordering::Relaxed), 0); + obj.set_mem_recorder(mem.clone()); + obj.record_mem_size(); + assert_eq!(mem.load(Ordering::Relaxed), obj.mem_size()); + drop(obj); + assert_eq!(mem.load(Ordering::Relaxed), 0); + } #[test] fn test_cache_object_with_same_size() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let a = CacheObject { - info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, data_decompressed: vec![0; 1024], mem_recorder: None, @@ -404,13 +424,30 @@ mod test { let b = ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(false)), None); assert!(b.heap_size() == 1024); } + #[test] + fn test_cache_object_with_same_size_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + assert!(a.heap_size() == 2048); + + // let b = ArcWrapper(Arc::new(a.clone())); + let b = ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(false)), None); + assert!(b.heap_size() == 2048); + } #[test] fn test_cache_object_with_lru() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut cache = LruCache::new(2048); - let hash_a = SHA1::default(); - let hash_b = SHA1::new(b"b"); // whatever different hash + let hash_a = ObjectHash::default(); + let hash_b = ObjectHash::new(b"b"); // whatever different hash let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a), offset: 0, @@ -434,6 +471,60 @@ mod test { ); assert!(r.is_ok()) } + { + let r = cache.try_insert( + hash_b.to_string(), + ArcWrapper::new(Arc::new(b.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_err()); + if let Err(lru_mem::TryInsertError::WouldEjectLru { .. }) = r { + // match the specified error, no extra action needed + } else { + panic!("Expected WouldEjectLru error"); + } + // insert b with a different key, so a will be ejected + let r = cache.insert( + hash_b.to_string(), + ArcWrapper::new(Arc::new(b.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_ok()); + } + { + // a should be ejected + let r = cache.get(&hash_a.to_string()); + assert!(r.is_none()); + } + } + #[test] + fn test_cache_object_with_lru_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut cache = LruCache::new(4096); + + let hash_a = ObjectHash::default(); + let hash_b = ObjectHash::new(b"b"); // whatever different hash + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + println!("a.heap_size() = {}", a.heap_size()); + + let b = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_b), + offset: 0, + data_decompressed: vec![0; 3072], + mem_recorder: None, + is_delta_in_pack: false, + }; + { + let r = cache.insert( + hash_a.to_string(), + ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_ok()) + } { let r = cache.try_insert( hash_b.to_string(), @@ -523,8 +614,9 @@ mod test { #[test] fn test_cache_object_serialize() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let a = CacheObject { - info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, data_decompressed: vec![0; 1024], mem_recorder: None, @@ -538,6 +630,24 @@ mod test { assert_eq!(a.data_decompressed, b.data_decompressed); assert_eq!(a.offset, b.offset); } + #[test] + fn test_cache_object_serialize_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + let s = bincode::serde::encode_to_vec(&a, bincode::config::standard()).unwrap(); + let b: CacheObject = bincode::serde::decode_from_slice(&s, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(a.info, b.info); + assert_eq!(a.data_decompressed, b.data_decompressed); + assert_eq!(a.offset, b.offset); + } #[test] fn test_arc_wrapper_drop_store() { diff --git a/src/internal/pack/decode.rs b/src/internal/pack/decode.rs index 1cd5ef47..92bcf666 100644 --- a/src/internal/pack/decode.rs +++ b/src/internal/pack/decode.rs @@ -14,7 +14,7 @@ use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::{ObjectHash, get_hash_kind, set_hash_kind}; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::zstdelta; @@ -79,7 +79,7 @@ impl Pack { let cache_mem_size = mem_limit.map(|mem_limit| mem_limit * 4 / 5); Pack { number: 0, - signature: SHA1::default(), + signature: ObjectHash::default(), objects: Vec::new(), pool: Arc::new(ThreadPool::new(thread_num)), waitlist: Arc::new(Waitlist::new()), @@ -309,10 +309,10 @@ impl Pack { }) } ObjectType::HashDelta => { - // Read 20 bytes to get the reference object SHA1 hash - let ref_sha1 = SHA1::from_stream(pack).unwrap(); - // Offset is incremented by 20 bytes - *offset += SHA1::SIZE; + // Read hash bytes to get the reference object hash(size depends on hash kind,e.g.,20 for SHA1,32 for SHA256) + let ref_sha = ObjectHash::from_stream(pack).unwrap(); + // Offset is incremented by 20/32 bytes + *offset += get_hash_kind().size(); let (data, raw_size) = Pack::decompress_data(pack, size)?; *offset += raw_size; @@ -321,7 +321,7 @@ impl Pack { let (_, final_size) = utils::read_delta_object_size(&mut reader)?; Ok(CacheObject { - info: CacheObjectInfo::HashDelta(ref_sha1, final_size), + info: CacheObjectInfo::HashDelta(ref_sha, final_size), offset: init_offset, data_decompressed: data, mem_recorder: None, @@ -345,7 +345,7 @@ impl Pack { ) -> Result<(), GitError> where F: Fn(MetaAttached) + Sync + Send + 'static, - C: FnOnce(SHA1) + Send + 'static, + C: FnOnce(ObjectHash) + Send + 'static, { let time = Instant::now(); let mut last_update_time = time.elapsed().as_millis(); @@ -414,7 +414,9 @@ impl Pack { let caches = caches.clone(); let waitlist = self.waitlist.clone(); + let kind = get_hash_kind(); self.pool.execute(move || { + set_hash_kind(kind); match obj.info { CacheObjectInfo::BaseObject(_, _) => { Self::cache_obj_and_process_waitlist(params, obj); @@ -454,7 +456,7 @@ impl Pack { } log_info(i, self); let render_hash = reader.final_hash(); - self.signature = SHA1::from_stream(&mut reader).unwrap(); + self.signature = ObjectHash::from_stream(&mut reader).unwrap(); if render_hash != self.signature { return Err(GitError::InvalidPackFile(format!( @@ -503,7 +505,9 @@ impl Pack { mut pack: impl BufRead + Send + 'static, sender: UnboundedSender, ) -> JoinHandle { + let kind = get_hash_kind(); thread::spawn(move || { + set_hash_kind(kind); self.decode( &mut pack, move |entry| { @@ -511,7 +515,7 @@ impl Pack { eprintln!("Channel full, failed to send entry: {e:?}"); } }, - None::, + None::, ) .unwrap(); self @@ -523,8 +527,9 @@ impl Pack { mut self, mut stream: impl Stream> + Unpin + Send + 'static, sender: UnboundedSender>, - pack_hash_send: Option>, + pack_hash_send: Option>, ) -> Self { + let kind = get_hash_kind(); let (tx, rx) = std::sync::mpsc::channel(); let mut reader = StreamBufReader::new(rx); tokio::spawn(async move { @@ -539,6 +544,7 @@ impl Pack { // CPU-bound task, so use spawn_blocking // DO NOT use thread::spawn, because it will block tokio runtime (if single-threaded runtime, like in tests) tokio::task::spawn_blocking(move || { + set_hash_kind(kind); self.decode( &mut reader, move |entry: MetaAttached| { @@ -547,7 +553,7 @@ impl Pack { eprintln!("unbound channel Sending Error: {e:?}"); } }, - Some(move |pack_id: SHA1| { + Some(move |pack_id: ObjectHash| { if let Some(pack_id_send) = pack_hash_send && let Err(e) = pack_id_send.send(pack_id) { @@ -733,7 +739,7 @@ mod tests { use flate2::write::ZlibEncoder; use tokio_util::io::ReaderStream; - use crate::hash::SHA1; + use crate::hash::{HashKind, ObjectHash, set_hash_kind}; use crate::internal::pack::Pack; use crate::internal::pack::tests::init_logger; use futures_util::TryStreamExt; @@ -783,7 +789,22 @@ mod tests { let f = fs::File::open(source).unwrap(); let mut buffered = BufReader::new(f); let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + } + #[test] + fn test_pack_decode_without_delta_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); } #[test] @@ -799,11 +820,28 @@ mod tests { let f = fs::File::open(source).unwrap(); let mut buffered = BufReader::new(f); let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + } + #[test] + fn test_pack_decode_with_ref_delta_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + init_logger(); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); } #[test] fn test_pack_decode_no_mem_limit() { + let _guard = set_hash_kind(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/pack-1d0e6c14760c956c173ede71cb28f33d921e232f.pack"); @@ -812,11 +850,27 @@ mod tests { let f = fs::File::open(source).unwrap(); let mut buffered = BufReader::new(f); let mut p = Pack::new(None, None, Some(tmp), true); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + } + #[test] + fn test_pack_decode_no_mem_limit_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, None, Some(tmp), true); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); } #[tokio::test] async fn test_pack_decode_with_large_file_with_delta_without_ref() { + let _guard = set_hash_kind(HashKind::Sha1); init_logger(); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack"); @@ -827,7 +881,7 @@ mod tests { let mut buffered = BufReader::new(f); let mut p = Pack::new( Some(20), - Some(1024 * 1024 * 1024 * 2), + Some(1024 * 1024 * 1024 * 1), //try to avoid dead lock on CI servers with low memory Some(tmp.clone()), true, ); @@ -836,7 +890,31 @@ mod tests { |_obj| { // println!("{:?} {}", obj.hash.to_string(), offset); }, - None::, + None::, + ); + if let Err(e) = rt { + fs::remove_dir_all(tmp).unwrap(); + panic!("Error: {e:?}"); + } + } // it will be stuck on dropping `Pack` on Windows if `mem_size` is None, so we need `mimalloc` + #[tokio::test] + async fn test_pack_decode_with_large_file_with_delta_without_ref_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + init_logger(); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(Some(20), Some(1024 * 1024), Some(tmp.clone()), true); + let rt = p.decode( + &mut buffered, + |_obj| { + // println!("{:?} {}", obj.hash.to_string(), offset); + }, + None::, ); if let Err(e) = rt { fs::remove_dir_all(tmp).unwrap(); @@ -855,7 +933,7 @@ mod tests { let stream = ReaderStream::new(f).map_err(axum::Error::new); let p = Pack::new( Some(20), - Some(1024 * 1024 * 1024 * 4), + Some(1024 * 1024 * 1024 * 1), Some(tmp.clone()), true, ); @@ -878,6 +956,36 @@ mod tests { assert_eq!(count.load(Ordering::Acquire), p.number); assert_eq!(p.number, 358109); } + #[tokio::test] + async fn test_decode_large_file_stream_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + init_logger(); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + let f = tokio::fs::File::open(source).await.unwrap(); + let stream = ReaderStream::new(f).map_err(axum::Error::new); + let p = Pack::new(Some(20), Some(1024 * 1024), Some(tmp.clone()), true); + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let handle = tokio::spawn(async move { p.decode_stream(stream, tx, None).await }); + let count = Arc::new(AtomicUsize::new(0)); + let count_c = count.clone(); + // in tests, RUNTIME is single-threaded, so `sync code` will block the tokio runtime + let consume = tokio::spawn(async move { + let mut cnt = 0; + while let Some(_entry) = rx.recv().await { + cnt += 1; + } + tracing::info!("Received: {}", cnt); + count_c.store(cnt, Ordering::Release); + }); + let p = handle.await.unwrap(); + consume.await.unwrap(); + assert_eq!(count.load(Ordering::Acquire), p.number); + assert_eq!(p.number, 26); + } #[tokio::test] async fn test_decode_large_file_async() { @@ -889,7 +997,7 @@ mod tests { let buffered = BufReader::new(f); let p = Pack::new( Some(20), - Some(1024 * 1024 * 1024 * 2), + Some(1024 * 1024 * 1024 * 1), Some(tmp.clone()), true, ); @@ -903,9 +1011,30 @@ mod tests { let p = handle.join().unwrap(); assert_eq!(cnt, p.number); } + #[tokio::test] + async fn test_decode_large_file_async_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + let f = fs::File::open(source).unwrap(); + let buffered = BufReader::new(f); + let p = Pack::new(Some(20), Some(1024 * 1024), Some(tmp.clone()), true); + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let handle = p.decode_async(buffered, tx); // new thread + let mut cnt = 0; + while let Some(_entry) = rx.recv().await { + cnt += 1; //use entry here + } + let p = handle.join().unwrap(); + assert_eq!(cnt, p.number); + } #[test] fn test_pack_decode_with_delta_without_ref() { + let _guard = set_hash_kind(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/pack-d50df695086eea6253a237cb5ac44af1629e7ced.pack"); @@ -915,12 +1044,30 @@ mod tests { let mut buffered = BufReader::new(f); let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); print!("pack_id: {:?}", p.signature); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + print!("pack_id: {:?}", p.signature.to_string()); + } + #[test] + fn test_pack_decode_with_delta_without_ref_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + print!("pack_id: {:?}", p.signature); //default sha1 + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); print!("pack_id: {:?}", p.signature.to_string()); } #[test] // Take too long time fn test_pack_decode_multi_task_with_large_file_with_delta_without_ref() { + let _guard = set_hash_kind(HashKind::Sha1); let task1 = std::thread::spawn(|| { test_pack_decode_with_large_file_with_delta_without_ref(); }); @@ -928,6 +1075,19 @@ mod tests { test_pack_decode_with_large_file_with_delta_without_ref(); }); + task1.join().unwrap(); + task2.join().unwrap(); + } + #[test] + fn test_pack_decode_multi_task_with_large_file_with_delta_without_ref_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let task1 = std::thread::spawn(|| { + test_pack_decode_with_large_file_with_delta_without_ref_sha256(); + }); + let task2 = std::thread::spawn(|| { + test_pack_decode_with_large_file_with_delta_without_ref_sha256(); + }); + task1.join().unwrap(); task2.join().unwrap(); } diff --git a/src/internal/pack/encode.rs b/src/internal/pack/encode.rs index 0f189977..43882528 100644 --- a/src/internal/pack/encode.rs +++ b/src/internal/pack/encode.rs @@ -7,12 +7,14 @@ use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::object::types::ObjectType; use crate::time_it; use crate::zstdelta; -use crate::{errors::GitError, hash::SHA1, internal::pack::entry::Entry}; +use crate::{ + errors::GitError, hash::ObjectHash, internal::pack::entry::Entry, utils::HashAlgorithm, +}; use ahash::AHasher; use flate2::write::ZlibEncoder; use natord::compare; use rayon::prelude::*; -use sha1::{Digest, Sha1}; + use std::hash::{Hash, Hasher}; use std::path::Path; use tokio::sync::mpsc; @@ -29,9 +31,9 @@ pub struct PackEncoder { window_size: usize, // window: VecDeque<(Entry, usize)>, // entry and offset sender: Option>>, - inner_offset: usize, // offset of current entry - inner_hash: Sha1, // Not SHA1 because need update trait - final_hash: Option, + inner_offset: usize, // offset of current entry + inner_hash: HashAlgorithm, // introduce different hash algorithm + final_hash: Option, start_encoding: bool, } @@ -184,8 +186,8 @@ impl PackEncoder { process_index: 0, // window: VecDeque::with_capacity(window_size), sender: Some(sender), - inner_offset: 12, // 12 bytes header - inner_hash: Sha1::new(), + inner_offset: 12, // start after 12 bytes pack header(signature + version + object count). + inner_hash: HashAlgorithm::new(), // introduce different hash algorithm final_hash: None, start_encoding: false, } @@ -202,7 +204,7 @@ impl PackEncoder { } /// Get the hash of the pack file. if the pack file is not finished, return None - pub fn get_hash(&self) -> Option { + pub fn get_hash(&self) -> Option { self.final_hash } @@ -341,7 +343,7 @@ impl PackEncoder { // Hash signature let hash_result = self.inner_hash.clone().finalize(); - self.final_hash = Some(SHA1::from_bytes(&hash_result)); + self.final_hash = Some(ObjectHash::from_bytes(&hash_result).unwrap()); self.send_data(hash_result.to_vec()).await; self.drop_sender(); @@ -535,7 +537,7 @@ impl PackEncoder { // hash signature let hash_result = self.inner_hash.clone().finalize(); - self.final_hash = Some(SHA1::from_bytes(&hash_result)); + self.final_hash = Some(ObjectHash::from_bytes(&hash_result).unwrap()); self.send_data(hash_result.to_vec()).await; self.drop_sender(); Ok(()) @@ -585,13 +587,13 @@ mod tests { use std::{io::Cursor, path::PathBuf}; use tokio::sync::Mutex; + use super::*; + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; use crate::internal::object::blob::Blob; use crate::internal::pack::utils::read_offset_encoding; use crate::internal::pack::{Pack, tests::init_logger}; use crate::time_it; - use super::*; - fn check_format(data: &Vec) { let mut p = Pack::new( None, @@ -601,12 +603,13 @@ mod tests { ); let mut reader = Cursor::new(data); tracing::debug!("start check format"); - p.decode(&mut reader, |_| {}, None::) + p.decode(&mut reader, |_| {}, None::) .expect("pack file format error"); } #[tokio::test] async fn test_pack_encoder() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); async fn encode_once(window_size: usize) -> Vec { let (tx, mut rx) = mpsc::channel(100); let (entry_tx, entry_rx) = mpsc::channel::>(1); @@ -646,6 +649,48 @@ mod tests { assert!(pack_with_delta.len() <= pack_without_delta_size); check_format(&pack_with_delta); } + #[tokio::test] + async fn test_pack_encoder_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + + async fn encode_once(window_size: usize) -> Vec { + let (tx, mut rx) = mpsc::channel(100); + let (entry_tx, entry_rx) = mpsc::channel::>(1); + + let str_vec = vec!["hello, word", "hello, world.", "!", "123141251251"]; + let encoder = PackEncoder::new(str_vec.len(), window_size, tx); + encoder.encode_async(entry_rx).await.unwrap(); + + for s in str_vec { + let blob = Blob::from_content(s); + let entry: Entry = blob.into(); + entry_tx + .send(MetaAttached { + inner: entry, + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + result + } + + // without delta + let pack_without_delta = encode_once(0).await; + let pack_without_delta_size = pack_without_delta.len(); + check_format(&pack_without_delta); + + // with delta + let pack_with_delta = encode_once(4).await; + assert!(pack_with_delta.len() <= pack_without_delta_size); + check_format(&pack_with_delta); + } async fn get_entries_for_test() -> Arc>> { let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -664,7 +709,33 @@ mod tests { let mut entries = entries_clone.blocking_lock(); entries.push(entry.inner); }, - None::, + None::, + ) + .unwrap(); + assert_eq!(p.number, entries.lock().await.len()); + tracing::info!("total entries: {}", p.number); + drop(p); + + entries + } + async fn get_entries_for_test_sha256() -> Arc>> { + let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let mut p = Pack::new(None, None, Some(PathBuf::from("/tmp/.cache_temp")), true); + + let f = std::fs::File::open(&source).unwrap(); + tracing::info!("pack file size: {}", f.metadata().unwrap().len()); + let mut reader = std::io::BufReader::new(f); + let entries = Arc::new(Mutex::new(Vec::new())); + let entries_clone = entries.clone(); + p.decode( + &mut reader, + move |entry| { + let mut entries = entries_clone.blocking_lock(); + entries.push(entry.inner); + }, + None::, ) .unwrap(); assert_eq!(p.number, entries.lock().await.len()); @@ -676,6 +747,7 @@ mod tests { #[tokio::test] async fn test_pack_encoder_parallel_large_file() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let start = Instant::now(); @@ -735,9 +807,70 @@ mod tests { // check format check_format(&result); } + #[tokio::test] + async fn test_pack_encoder_parallel_large_file_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + + let start = Instant::now(); + // use sha256 pack file for testing + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let (tx, mut rx) = mpsc::channel(1_000_000); + let (entry_tx, entry_rx) = mpsc::channel::>(1_000_000); + + let mut encoder = PackEncoder::new(entries_number, 0, tx); + tokio::spawn(async move { + time_it!("test parallel encode sha256", { + encoder.parallel_encode(entry_rx).await.unwrap(); + }); + }); + + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("sha256 test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", result.len()); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + check_format(&result); + } #[tokio::test] async fn test_pack_encoder_large_file() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let entries = get_entries_for_test().await; let entries_number = entries.lock().await.len(); @@ -804,9 +937,79 @@ mod tests { total_original_size.saturating_sub(pack_size) ); } + #[tokio::test] + async fn test_pack_encoder_large_file_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let start = Instant::now(); + // encode entries + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::>(100_000); + + let mut encoder = PackEncoder::new(entries_number, 0, tx); + tokio::spawn(async move { + time_it!("test encode no parallel sha256", { + encoder.encode(entry_rx).await.unwrap(); + }); + }); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + // // only receive data + // while (rx.recv().await).is_some() { + // // do nothing + // } + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + } #[tokio::test] async fn test_pack_encoder_with_zstdelta() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let entries = get_entries_for_test().await; let entries_number = entries.lock().await.len(); @@ -866,6 +1069,68 @@ mod tests { // check format check_format(&result); } + #[tokio::test] + async fn test_pack_encoder_with_zstdelta_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let start = Instant::now(); + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::>(100_000); + + let encoder = PackEncoder::new(entries_number, 10, tx); + encoder.encode_async_with_zstdelta(entry_rx).await.unwrap(); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + + // check format + check_format(&result); + } #[test] fn test_encode_offset() { @@ -882,6 +1147,7 @@ mod tests { #[tokio::test] async fn test_pack_encoder_large_file_with_delta() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let entries = get_entries_for_test().await; let entries_number = entries.lock().await.len(); @@ -939,6 +1205,69 @@ mod tests { total_original_size.saturating_sub(pack_size) ); + // check format + check_format(&result); + } + #[tokio::test] + async fn test_pack_encoder_large_file_with_delta_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::>(100_000); + + let encoder = PackEncoder::new(entries_number, 10, tx); + + let start = Instant::now(); // 开始时间 + encoder.encode_async(entry_rx).await.unwrap(); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + // check format check_format(&result); } diff --git a/src/internal/pack/entry.rs b/src/internal/pack/entry.rs index f18269e1..627d1e01 100644 --- a/src/internal/pack/entry.rs +++ b/src/internal/pack/entry.rs @@ -2,7 +2,7 @@ use std::hash::{Hash, Hasher}; use serde::{Deserialize, Serialize}; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::blob::Blob; use crate::internal::object::commit::Commit; @@ -17,7 +17,7 @@ use crate::internal::object::types::ObjectType; pub struct Entry { pub obj_type: ObjectType, pub data: Vec, - pub hash: SHA1, + pub hash: ObjectHash, pub chain_len: usize, } diff --git a/src/internal/pack/mod.rs b/src/internal/pack/mod.rs index 3307f838..a4f71f9a 100644 --- a/src/internal/pack/mod.rs +++ b/src/internal/pack/mod.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicUsize; use threadpool::ThreadPool; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::pack::cache::Caches; use crate::internal::pack::waitlist::Waitlist; @@ -24,7 +24,7 @@ use crate::internal::pack::waitlist::Waitlist; const DEFAULT_TMP_DIR: &str = "./.cache_temp"; pub struct Pack { pub number: usize, - pub signature: SHA1, + pub signature: ObjectHash, pub objects: Vec>, pub pool: Arc, pub waitlist: Arc, diff --git a/src/internal/pack/utils.rs b/src/internal/pack/utils.rs index d98db98a..2a653146 100644 --- a/src/internal/pack/utils.rs +++ b/src/internal/pack/utils.rs @@ -3,7 +3,7 @@ use std::fs; use std::io::{self, Read}; use std::path::Path; -use crate::hash::SHA1; +use crate::hash::{ObjectHash, get_hash_kind}; use crate::internal::object::types::ObjectType; /// Checks if the reader has reached EOF (end of file). @@ -258,19 +258,36 @@ pub fn read_delta_object_size(stream: &mut R) -> io::Result<(usize, usi /// Calculate the SHA1 hash of the given object. ///
"` \0`" ///
data: The decompressed content of the object -pub fn calculate_object_hash(obj_type: ObjectType, data: &Vec) -> SHA1 { - let mut hash = Sha1::new(); - // Header: " \0" - hash.update(obj_type.to_bytes()); - hash.update(b" "); - hash.update(data.len().to_string()); - hash.update(b"\0"); - - // Decompressed data(raw content) - hash.update(data); - - let re: [u8; 20] = hash.finalize().into(); - SHA1(re) +pub fn calculate_object_hash(obj_type: ObjectType, data: &Vec) -> ObjectHash { + match get_hash_kind() { + crate::hash::HashKind::Sha1 => { + let mut hash = Sha1::new(); + // Header: " \0" + hash.update(obj_type.to_bytes()); + hash.update(b" "); + hash.update(data.len().to_string()); + hash.update(b"\0"); + + // Decompressed data(raw content) + hash.update(data); + + let re: [u8; 20] = hash.finalize().into(); + ObjectHash::Sha1(re) + } + crate::hash::HashKind::Sha256 => { + let mut hash = sha2::Sha256::new(); + // Header: " \0" + hash.update(obj_type.to_bytes()); + hash.update(b" "); + hash.update(data.len().to_string()); + hash.update(b"\0"); + + // Decompressed data(raw content) + hash.update(data); + let re: [u8; 32] = hash.finalize().into(); + ObjectHash::Sha256(re) + } + } } /// Create an empty directory or clear the existing directory. pub fn create_empty_dir>(path: P) -> io::Result<()> { @@ -314,6 +331,7 @@ macro_rules! time_it { #[cfg(test)] mod tests { + use crate::hash::{HashKind, set_hash_kind_for_test}; use crate::internal::object::types::ObjectType; use std::io; use std::io::Cursor; @@ -323,9 +341,19 @@ mod tests { #[test] fn test_calc_obj_hash() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash = calculate_object_hash(ObjectType::Blob, &b"a".to_vec()); assert_eq!(hash.to_string(), "2e65efe2a145dda7ee51d1741299f848e5bf752e"); } + #[test] + fn test_calc_obj_hash_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash = calculate_object_hash(ObjectType::Blob, &b"a".to_vec()); + assert_eq!( + hash.to_string(), + "eb337bcee2061c5313c9a1392116b6c76039e9e30d71467ae359b36277e17dc7" + ); + } #[test] fn eof() { diff --git a/src/internal/pack/waitlist.rs b/src/internal/pack/waitlist.rs index 13107b79..a678ac6f 100644 --- a/src/internal/pack/waitlist.rs +++ b/src/internal/pack/waitlist.rs @@ -1,4 +1,4 @@ -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::pack::cache_object::CacheObject; use dashmap::DashMap; @@ -8,7 +8,7 @@ use dashmap::DashMap; pub struct Waitlist { //TODO Memory Control! pub map_offset: DashMap>, - pub map_ref: DashMap>, + pub map_ref: DashMap>, } impl Waitlist { @@ -20,13 +20,13 @@ impl Waitlist { self.map_offset.entry(offset).or_default().push(obj); } - pub fn insert_ref(&self, hash: SHA1, obj: CacheObject) { + pub fn insert_ref(&self, hash: ObjectHash, obj: CacheObject) { self.map_ref.entry(hash).or_default().push(obj); } /// Take objects out (get & remove) ///
Return Vec::new() if None - pub fn take(&self, offset: usize, hash: SHA1) -> Vec { + pub fn take(&self, offset: usize, hash: ObjectHash) -> Vec { let mut res = Vec::new(); if let Some((_, vec)) = self.map_offset.remove(&offset) { res.extend(vec); diff --git a/src/internal/pack/wrapper.rs b/src/internal/pack/wrapper.rs index 057d77a0..717cc352 100644 --- a/src/internal/pack/wrapper.rs +++ b/src/internal/pack/wrapper.rs @@ -2,19 +2,20 @@ use std::io::{self, BufRead, Read}; use sha1::{Digest, Sha1}; -use crate::hash::SHA1; - -/// [`Wrapper`] is a wrapper around a reader that also computes the SHA1 hash of the data read. +use crate::hash::{HashKind, ObjectHash, get_hash_kind}; +use crate::utils::HashAlgorithm; +/// [`Wrapper`] is a wrapper around a reader that also computes the SHA1/ SHA256 hash of the data read. /// /// It is designed to work with any reader that implements `BufRead`. /// /// Fields: /// * `inner`: The inner reader. -/// * `hash`: The SHA1 hash state. +/// * `hash`: The hash state. /// * `count_hash`: A flag to indicate whether to compute the hash while reading. +/// pub struct Wrapper { inner: R, - hash: Sha1, + hash: HashAlgorithm, bytes_read: usize, } @@ -30,7 +31,10 @@ where pub fn new(inner: R) -> Self { Self { inner, - hash: Sha1::new(), // Initialize a new SHA1 hasher + hash: match get_hash_kind() { + HashKind::Sha1 => HashAlgorithm::Sha1(Sha1::new()), + HashKind::Sha256 => HashAlgorithm::Sha256(sha2::Sha256::new()), + }, // Initialize a new SHA1/ SHA256 hasher bytes_read: 0, } } @@ -39,12 +43,20 @@ where self.bytes_read } - /// Returns the final SHA1 hash of the data read so far. + /// Returns the final SHA1/ SHA256 hash of the data read so far. /// - /// This is a clone of the internal hash state finalized into a SHA1 hash. - pub fn final_hash(&self) -> SHA1 { - let re: [u8; 20] = self.hash.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes - SHA1(re) + /// This is a clone of the internal hash state finalized into a SHA1/ SHA256 hash. + pub fn final_hash(&self) -> ObjectHash { + match &self.hash.clone() { + HashAlgorithm::Sha1(hasher) => { + let re: [u8; 20] = hasher.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes + ObjectHash::from_bytes(&re).unwrap() + } + HashAlgorithm::Sha256(hasher) => { + let re: [u8; 32] = hasher.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes + ObjectHash::from_bytes(&re).unwrap() + } + } } } @@ -63,7 +75,10 @@ where /// * `amt`: The amount of data to consume from the buffer. fn consume(&mut self, amt: usize) { let buffer = self.inner.fill_buf().expect("Failed to fill buffer"); - self.hash.update(&buffer[..amt]); // Update hash with the data being consumed + match &mut self.hash { + HashAlgorithm::Sha1(hasher) => hasher.update(&buffer[..amt]), // Update SHA1 hash with the data being consumed + HashAlgorithm::Sha256(hasher) => hasher.update(&buffer[..amt]), // Update SHA256 hash with the data being consumed + } self.inner.consume(amt); // Consume the data from the inner reader self.bytes_read += amt; } @@ -83,7 +98,10 @@ where /// Returns the number of bytes read. fn read(&mut self, buf: &mut [u8]) -> io::Result { let o = self.inner.read(buf)?; // Read data into the buffer - self.hash.update(&buf[..o]); // Update hash with the data being read + match &mut self.hash { + HashAlgorithm::Sha1(hasher) => hasher.update(&buf[..o]), // Update SHA1 hash with the data being read + HashAlgorithm::Sha256(hasher) => hasher.update(&buf[..o]), // Update SHA256 hash with the data being read + } self.bytes_read += o; Ok(o) // Return the number of bytes read } @@ -95,10 +113,25 @@ mod tests { use sha1::{Digest, Sha1}; + use crate::hash::{HashKind, set_hash_kind}; use crate::internal::pack::wrapper::Wrapper; - #[test] fn test_wrapper_read() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha1); + let data = b"Hello, world!"; // Sample data + let cursor = Cursor::new(data.as_ref()); + let buf_reader = BufReader::new(cursor); + let mut wrapper = Wrapper::new(buf_reader); + + let mut buffer = vec![0; data.len()]; + wrapper.read_exact(&mut buffer)?; + + assert_eq!(buffer, data); + Ok(()) + } + #[test] + fn test_wrapper_read_sha256() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha256); let data = b"Hello, world!"; // Sample data let cursor = Cursor::new(data.as_ref()); let buf_reader = BufReader::new(cursor); @@ -113,6 +146,7 @@ mod tests { #[test] fn test_wrapper_hash() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha1); let data = b"Hello, world!"; let cursor = Cursor::new(data.as_ref()); let buf_reader = BufReader::new(cursor); @@ -126,7 +160,26 @@ mod tests { hasher.update(data); let expected_hash: [u8; 20] = hasher.finalize().into(); - assert_eq!(hash_result.0, expected_hash); + assert_eq!(hash_result.as_ref(), expected_hash); + Ok(()) + } + #[test] + fn test_wrapper_hash_sha256() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha256); + let data = b"Hello, world!"; + let cursor = Cursor::new(data.as_ref()); + let buf_reader = BufReader::new(cursor); + let mut wrapper = Wrapper::new(buf_reader); + + let mut buffer = vec![0; data.len()]; + wrapper.read_exact(&mut buffer)?; + + let hash_result = wrapper.final_hash(); + let mut hasher = sha2::Sha256::new(); + hasher.update(data); + let expected_hash: [u8; 32] = hasher.finalize().into(); + + assert_eq!(hash_result.as_ref(), &expected_hash); Ok(()) } } diff --git a/src/protocol/core.rs b/src/protocol/core.rs index 7bdf0a63..2bfe2b27 100644 --- a/src/protocol/core.rs +++ b/src/protocol/core.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use bytes::Bytes; use futures::stream::StreamExt; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::protocol::smart::SmartProtocol; @@ -63,7 +63,7 @@ pub trait RepositoryAccess: Send + Sync + Clone { object_hash: &str, ) -> Result { let data = self.get_object(object_hash).await?; - let hash = SHA1::from_str(object_hash) + let hash = ObjectHash::from_str(object_hash) .map_err(|e| ProtocolError::repository_error(format!("Invalid hash format: {}", e)))?; crate::internal::object::blob::Blob::from_bytes(&data, hash) @@ -79,7 +79,7 @@ pub trait RepositoryAccess: Send + Sync + Clone { commit_hash: &str, ) -> Result { let data = self.get_object(commit_hash).await?; - let hash = SHA1::from_str(commit_hash) + let hash = ObjectHash::from_str(commit_hash) .map_err(|e| ProtocolError::repository_error(format!("Invalid hash format: {}", e)))?; crate::internal::object::commit::Commit::from_bytes(&data, hash) @@ -95,7 +95,7 @@ pub trait RepositoryAccess: Send + Sync + Clone { tree_hash: &str, ) -> Result { let data = self.get_object(tree_hash).await?; - let hash = SHA1::from_str(tree_hash) + let hash = ObjectHash::from_str(tree_hash) .map_err(|e| ProtocolError::repository_error(format!("Invalid hash format: {}", e)))?; crate::internal::object::tree::Tree::from_bytes(&data, hash) diff --git a/src/protocol/pack.rs b/src/protocol/pack.rs index 3083d530..e6f5e358 100644 --- a/src/protocol/pack.rs +++ b/src/protocol/pack.rs @@ -1,6 +1,6 @@ use super::core::RepositoryAccess; use super::types::ProtocolError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::object::types::ObjectType; use crate::internal::object::{ObjectTrait, blob::Blob, commit::Commit, tree::Tree}; @@ -126,7 +126,7 @@ where tracing::warn!("Unknown object type in pack: {:?}", entry.inner.obj_type); } }, - None::, + None::, ) .map_err(|e| ProtocolError::invalid_request(&format!("Failed to decode pack: {}", e)))?; @@ -348,6 +348,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; use crate::internal::object::blob::Blob; use crate::internal::object::commit::Commit; use crate::internal::object::signature::{Signature, SignatureType}; @@ -399,6 +400,98 @@ mod tests { #[tokio::test] async fn test_pack_roundtrip_encode_decode() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); + // Create two Blob objects + let blob1 = Blob::from_content("hello"); + let blob2 = Blob::from_content("world"); + + // Create a Tree containing two file items + let item1 = TreeItem::new(TreeItemMode::Blob, blob1.id, "hello.txt".to_string()); + let item2 = TreeItem::new(TreeItemMode::Blob, blob2.id, "world.txt".to_string()); + let tree = Tree::from_tree_items(vec![item1, item2]).unwrap(); + + // Create a Commit pointing to the Tree + let author = Signature::new( + SignatureType::Author, + "tester".to_string(), + "tester@example.com".to_string(), + ); + let committer = Signature::new( + SignatureType::Committer, + "tester".to_string(), + "tester@example.com".to_string(), + ); + let commit = Commit::new(author, committer, tree.id, vec![], "init commit"); + + // Generate pack stream + let (tx, mut rx) = mpsc::channel::>(64); + PackGenerator::::generate_pack_stream( + ( + vec![commit.clone()], + vec![tree.clone()], + vec![blob1.clone(), blob2.clone()], + ), + tx, + ) + .await + .unwrap(); + + let mut pack_bytes: Vec = Vec::new(); + while let Some(chunk) = rx.recv().await { + pack_bytes.extend_from_slice(&chunk); + } + println!("Encoded pack size: {} bytes", pack_bytes.len()); + + // Unpack the pack stream + let dummy = DummyRepoAccess; + let generator = PackGenerator::new(&dummy); + let (decoded_commits, decoded_trees, decoded_blobs) = generator + .unpack_stream(Bytes::from(pack_bytes)) + .await + .unwrap(); + + println!( + "Decoded commits: {:?}", + decoded_commits + .iter() + .map(|c| c.id.to_string()) + .collect::>() + ); + println!( + "Decoded trees: {:?}", + decoded_trees + .iter() + .map(|t| t.id.to_string()) + .collect::>() + ); + println!( + "Decoded blobs: {:?}", + decoded_blobs + .iter() + .map(|b| b.id.to_string()) + .collect::>() + ); + + // Verify object ID roundtrip consistency + assert_eq!(decoded_commits.len(), 1); + assert_eq!(decoded_trees.len(), 1); + assert_eq!(decoded_blobs.len(), 2); + + assert_eq!(decoded_commits[0].id, commit.id); + assert_eq!(decoded_trees[0].id, tree.id); + + let mut orig_blob_ids = vec![blob1.id.to_string(), blob2.id.to_string()]; + orig_blob_ids.sort(); + let mut decoded_blob_ids = decoded_blobs + .iter() + .map(|b| b.id.to_string()) + .collect::>(); + decoded_blob_ids.sort(); + assert_eq!(orig_blob_ids, decoded_blob_ids); + } + #[tokio::test] + async fn test_pack_roundtrip_encode_decode_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); // Create two Blob objects let blob1 = Blob::from_content("hello"); let blob2 = Blob::from_content("world"); diff --git a/src/protocol/smart.rs b/src/protocol/smart.rs index 546a63af..0f2855f7 100644 --- a/src/protocol/smart.rs +++ b/src/protocol/smart.rs @@ -8,10 +8,9 @@ use super::types::ProtocolError; use super::types::{ COMMON_CAP_LIST, Capability, LF, NUL, PKT_LINE_END_MARKER, ProtocolStream, RECEIVE_CAP_LIST, RefCommand, RefTypeEnum, SP, ServiceType, SideBand, TransportProtocol, UPLOAD_CAP_LIST, - ZERO_ID, }; use super::utils::{add_pkt_line_string, build_smart_reply, read_pkt_line, read_until_white_space}; - +use crate::hash::{HashKind, ObjectHash, get_hash_kind}; /// Smart Git Protocol implementation /// /// This struct handles the Git smart protocol operations for both HTTP and SSH transports. @@ -25,7 +24,9 @@ where pub capabilities: Vec, pub side_band: Option, pub command_list: Vec, - + pub wire_hash_kind: HashKind, + pub local_hash_kind: HashKind, + pub zero_id: String, // Trait-based dependencies repo_storage: R, auth_service: A, @@ -36,6 +37,11 @@ where R: RepositoryAccess, A: AuthenticationService, { + pub fn set_wire_hash_kind(&mut self, kind: HashKind) { + self.wire_hash_kind = kind; + self.zero_id = ObjectHash::zero_str(kind); + } + /// Create a new SmartProtocol instance pub fn new(transport_protocol: TransportProtocol, repo_storage: R, auth_service: A) -> Self { Self { @@ -45,6 +51,9 @@ where command_list: Vec::new(), repo_storage, auth_service, + wire_hash_kind: HashKind::default(), // Default to SHA-1 + local_hash_kind: get_hash_kind(), + zero_id: ObjectHash::zero_str(HashKind::default()), } } @@ -81,7 +90,17 @@ where self.repo_storage.get_repository_refs().await.map_err(|e| { ProtocolError::repository_error(format!("Failed to get refs: {}", e)) })?; - + let hex_len = self.wire_hash_kind.hex_len(); + for (name, h) in &refs { + if h.len() != hex_len { + return Err(ProtocolError::invalid_request(&format!( + "Hash length mismatch for ref {}: expected {}, got {}", + name, + hex_len, + h.len() + ))); + } + } // Ensure refs match the expected wire hash kind // Convert to the expected format (head_hash, git_refs) let head_hash = refs .iter() @@ -89,21 +108,25 @@ where name == "HEAD" || name.ends_with("/main") || name.ends_with("/master") }) .map(|(_, hash)| hash.clone()) - .unwrap_or_else(|| "0000000000000000000000000000000000000000".to_string()); + .unwrap_or_else(|| self.zero_id.clone()); let git_refs: Vec = refs .into_iter() .map(|(name, hash)| super::types::GitRef { name, hash }) .collect(); - + // capability add object-format，declare the wire hash kind + let format_cap = match self.wire_hash_kind { + HashKind::Sha1 => " object-format=sha1", + HashKind::Sha256 => " object-format=sha256", + }; // Determine capabilities based on service type let cap_list = match service_type { - ServiceType::UploadPack => format!("{UPLOAD_CAP_LIST}{COMMON_CAP_LIST}"), - ServiceType::ReceivePack => format!("{RECEIVE_CAP_LIST}{COMMON_CAP_LIST}"), + ServiceType::UploadPack => format!("{UPLOAD_CAP_LIST}{COMMON_CAP_LIST}{format_cap}"), + ServiceType::ReceivePack => format!("{RECEIVE_CAP_LIST}{COMMON_CAP_LIST}{format_cap}"), }; // The stream MUST include capability declarations behind a NUL on the first ref. - let name = if head_hash == ZERO_ID { + let name = if head_hash == self.zero_id { "capabilities^{}" } else { "HEAD" @@ -275,8 +298,8 @@ where for command in &mut self.command_list { if command.ref_type == RefTypeEnum::Tag { // Just update if refs type is tag - // Convert ZERO_ID to None for old hash - let old_hash = if command.old_hash == ZERO_ID { + // Convert zero_id to None for old hash + let old_hash = if command.old_hash == self.zero_id { None } else { Some(command.old_hash.as_str()) @@ -294,8 +317,8 @@ where command.default_branch = true; default_exist = true; } - // Convert ZERO_ID to None for old hash - let old_hash = if command.old_hash == ZERO_ID { + // Convert zero_id to None for old hash + let old_hash = if command.old_hash == self.zero_id { None } else { Some(command.old_hash.as_str()) @@ -339,6 +362,16 @@ where /// Parse capabilities from capability string pub fn parse_capabilities(&mut self, cap_str: &str) { for cap in cap_str.split_whitespace() { + if let Some(fmt) = cap.strip_prefix("object-format=") { + match fmt { + "sha1" => self.set_wire_hash_kind(HashKind::Sha1), + "sha256" => self.set_wire_hash_kind(HashKind::Sha256), + _ => { + tracing::warn!("Unknown object-format capability: {}", fmt); + } + } + continue; + } if let Ok(capability) = cap.parse::() { self.capabilities.push(capability); } @@ -359,13 +392,14 @@ where #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::object::blob::Blob; use crate::internal::object::commit::Commit; use crate::internal::object::signature::{Signature, SignatureType}; use crate::internal::object::tree::{Tree, TreeItem, TreeItemMode}; use crate::internal::pack::{encode::PackEncoder, entry::Entry}; - use crate::protocol::types::{RefCommand, ZERO_ID}; // import sibling types + use crate::protocol::types::RefCommand; // import sibling types use crate::protocol::utils; // import sibling module use async_trait::async_trait; use bytes::Bytes; @@ -493,6 +527,7 @@ mod tests { #[tokio::test] async fn test_receive_pack_stream_status_report() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); // Build simple objects let blob1 = Blob::from_content("hello"); let blob2 = Blob::from_content("world"); @@ -565,8 +600,9 @@ mod tests { let repo_access = TestRepoAccess::new(); let auth = TestAuth; let mut smart = SmartProtocol::new(TransportProtocol::Http, repo_access.clone(), auth); + smart.set_wire_hash_kind(HashKind::Sha1); smart.command_list.push(RefCommand::new( - ZERO_ID.to_string(), + smart.zero_id.to_string(), commit.id.to_string(), "refs/heads/main".to_string(), )); @@ -599,4 +635,24 @@ mod tests { assert_eq!(repo_access.updates_len(), 1); assert!(repo_access.post_hook_called()); } + + #[tokio::test] + async fn info_refs_rejects_sha256_with_sha1_refs() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); // avoid thread-local contamination + let repo_access = TestRepoAccess::new(); // still returns 40-char strings + let auth = TestAuth; + let mut smart = SmartProtocol::new(TransportProtocol::Http, repo_access, auth); + smart.set_wire_hash_kind(HashKind::Sha256); // claims wire uses SHA-256 + // expect failure because refs are SHA-1 + let res = smart.git_info_refs(ServiceType::UploadPack).await; + assert!(res.is_err(), "expected failure when hash lengths mismatch"); + + smart.set_wire_hash_kind(HashKind::Sha1); + + let res = smart.git_info_refs(ServiceType::UploadPack).await; + assert!( + res.is_ok(), + "expected SHA1 refs to be accepted when wire is SHA1" + ); + } } diff --git a/src/protocol/types.rs b/src/protocol/types.rs index 14a8f5b1..fb62c674 100644 --- a/src/protocol/types.rs +++ b/src/protocol/types.rs @@ -364,9 +364,6 @@ pub enum CommandType { Delete, } -/// Zero object ID constant -pub const ZERO_ID: &str = "0000000000000000000000000000000000000000"; - /// Protocol constants pub const LF: char = '\n'; pub const SP: char = ' '; diff --git a/src/utils.rs b/src/utils.rs index 2ed62f41..dfaea03c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,15 +1,15 @@ -use crate::hash::SHA1; +use crate::hash::{HashKind, ObjectHash, get_hash_kind}; +use sha1::{Digest, Sha1}; use std::io; use std::io::{BufRead, Read}; - pub fn read_bytes(file: &mut impl Read, len: usize) -> io::Result> { let mut buf = vec![0; len]; file.read_exact(&mut buf)?; Ok(buf) } -pub fn read_sha1(file: &mut impl Read) -> io::Result { - SHA1::from_stream(file) +pub fn read_sha(file: &mut impl Read) -> io::Result { + ObjectHash::from_stream(file) } /// A lightweight wrapper that counts bytes read from the underlying reader. @@ -47,3 +47,48 @@ impl BufRead for CountingReader { self.inner.consume(amt); } } +/// a hash abstraction to support both SHA1 and SHA256 +/// which for stream hashing handle use (e.g. Sha1::new()) +/// `std::io::Write` trait to update the hash state +#[derive(Clone)] +pub enum HashAlgorithm { + Sha1(Sha1), + Sha256(sha2::Sha256), + // Future: support other hash algorithms +} +impl HashAlgorithm { + /// Update hash with data + pub fn update(&mut self, data: &[u8]) { + match self { + HashAlgorithm::Sha1(hasher) => hasher.update(data), + HashAlgorithm::Sha256(hasher) => hasher.update(data), + } + } + /// Finalize and get hash result + pub fn finalize(self) -> Vec { + match self { + HashAlgorithm::Sha1(hasher) => hasher.finalize().to_vec(), + HashAlgorithm::Sha256(hasher) => hasher.finalize().to_vec(), + } + } + pub fn new() -> Self { + match get_hash_kind() { + HashKind::Sha1 => HashAlgorithm::Sha1(Sha1::new()), + HashKind::Sha256 => HashAlgorithm::Sha256(sha2::Sha256::new()), + } + } +} +impl std::io::Write for HashAlgorithm { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.update(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +impl Default for HashAlgorithm { + fn default() -> Self { + Self::new() + } +} diff --git a/tests/data/index/index-9-256 b/tests/data/index/index-9-256 new file mode 100644 index 00000000..3e6b8b7d Binary files /dev/null and b/tests/data/index/index-9-256 differ diff --git a/tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack b/tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack new file mode 100644 index 00000000..e8f97153 --- /dev/null +++ b/tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bdad72b0d7ee8a375526ef57efb10963924fa484a8900e00a66481d8f40eb19 +size 20293 diff --git a/tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack b/tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack new file mode 100644 index 00000000..62843b81 --- /dev/null +++ b/tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9287eaf658f377eeb9956a17555b214291398bdb36c40d0527180a6c707dcce +size 1039 diff --git a/tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack b/tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack new file mode 100644 index 00000000..6be61a44 --- /dev/null +++ b/tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df0a36c6ca8c7d4324848c4502f1918e44a04a359b6f8ed255b296c91ed89821 +size 23190 diff --git a/tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack b/tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack new file mode 100644 index 00000000..c2d3f9da --- /dev/null +++ b/tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe0cea185b8622455c0d66089e693b4fabc662ac3d82dd7b10d5815fd43ba31 +size 4904