From d612fe6ec2976a7d12b63a7bf31e344ba846003b Mon Sep 17 00:00:00 2001 From: jackieismpc Date: Tue, 25 Nov 2025 15:50:27 +0800 Subject: [PATCH 1/7] 1.Introduce SHA-256 support using two abstraction layers 2.Refactor the tests in hash.rs 3. Import a pack file for SHA-256 testing Signed-off-by: jackieismpc --- Cargo.lock | 1 + Cargo.toml | 1 + src/hash.rs | 423 ++++++++++++++---- ...deb773fefc821f6f60f4f4797644ad43dad3d.pack | 3 + 4 files changed, 332 insertions(+), 96 deletions(-) create mode 100644 tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack diff --git a/Cargo.lock b/Cargo.lock index bf5c745a..86e63181 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1290,6 +1290,7 @@ dependencies = [ "serde", "serde_json", "sha1", + "sha2", "similar", "tempfile", "thiserror 2.0.17", diff --git a/Cargo.toml b/Cargo.toml index a3f7bd3c..6cb58b9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ natord = "1.0.9" tempfile = "3.23.0" path-absolutize = "3.1.1" similar = "2.7.0" +sha2 = "0.10.9" [dev-dependencies] diff --git a/src/hash.rs b/src/hash.rs index 9d22ca89..51612d6f 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -3,30 +3,21 @@ //! location in the Git internal and mega database. //! -use std::{fmt::Display, io}; +use std::{cell::RefCell, fmt::Display, io, str::FromStr}; +use crate::internal::object::types::ObjectType; use bincode::{Decode, Encode}; use colored::Colorize; use serde::{Deserialize, Serialize}; use sha1::Digest; -use crate::internal::object::types::ObjectType; - /// The [`SHA1`] struct, encapsulating a `[u8; 20]` array, is specifically designed to represent Git hash IDs. /// In Git's context, these IDs are 40-character hexadecimal strings generated via the SHA-1 algorithm. /// Each Git object receives a unique hash ID based on its content, serving as an identifier for its location /// within the Git internal database. Utilizing a dedicated struct for these hash IDs enhances code readability and /// maintainability by providing a clear, structured format for their manipulation and storage. /// -/// ### Change Log -/// -/// In previous versions of the 'mega' project, `Hash` was used to denote hash values. However, in newer versions, -/// `SHA1` is employed for this purpose. Future updates plan to extend support to SHA256 and SHA512, or potentially -/// other hash algorithms. By abstracting the hash model to `Hash`, and using specific imports like `use crate::hash::SHA1` -/// or `use crate::hash::SHA256`, the codebase maintains a high level of clarity and maintainability. This design choice -/// allows for easier adaptation to different hash algorithms while keeping the underlying implementation consistent and -/// understandable. - Nov 26, 2023 (by @genedna) -/// +/// The [`HashKind`] enum represents different types of hash algorithms supported in Git, #[derive( Clone, Copy, @@ -42,103 +33,195 @@ use crate::internal::object::types::ObjectType; Encode, Decode, )] -pub struct SHA1(pub [u8; 20]); +pub enum HashKind { + #[default] + Sha1, + Sha256, +} +/// Implementation of methods for the [`HashKind`] enum. +impl HashKind { + pub const fn size(&self) -> usize { + match self { + HashKind::Sha1 => 20, + HashKind::Sha256 => 32, + // Add more hash kinds here as needed + } + } + pub const fn hex_len(&self) -> usize { + match self { + HashKind::Sha1 => 40, + HashKind::Sha256 => 64, + } + } + pub const fn as_str(&self) -> &'static str { + match self { + HashKind::Sha1 => "sha1", + HashKind::Sha256 => "sha256", + } + } +} +impl std::fmt::Display for HashKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} +impl std::str::FromStr for HashKind { + type Err = String; -/// Display trait for SHA1. -impl Display for SHA1 { - /// Allows [`SHA1::to_string()`] to be used. - /// Note: If you want a terminal-friendly colorized output, use [`SHA1::to_color_str()`]. - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", hex::encode(self.0)) + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "sha1" => Ok(HashKind::Sha1), + "sha256" => Ok(HashKind::Sha256), + _ => Err("Invalid hash kind".to_string()), + } } } -impl AsRef<[u8]> for SHA1 { +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Deserialize, Serialize, Encode, Decode, +)] +pub enum ObjectHash { + Sha1([u8; 20]), + Sha256([u8; 32]), +} +impl Default for ObjectHash { + fn default() -> Self { + ObjectHash::Sha1([0u8; 20]) + } +} +impl Display for ObjectHash { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", hex::encode(self.as_ref())) + } +} +impl AsRef<[u8]> for ObjectHash { fn as_ref(&self) -> &[u8] { - &self.0 + match self { + ObjectHash::Sha1(bytes) => bytes.as_slice(), + ObjectHash::Sha256(bytes) => bytes.as_slice(), + } } } -/// Implementation of the [`std::str::FromStr`] trait for the [`SHA1`] type. -/// -/// To effectively use the `from_str` method for converting a string to a `SHA1` object, consider the following: -/// 1. The input string `s` should be a pre-calculated hexadecimal string, exactly 40 characters in length. This string -/// represents a SHA1 hash and should conform to the standard SHA1 hash format. +/// Implementation of the [`std::str::FromStr`] trait for the [`ObjectHash`] enum. +/// To effectively use the `from_str` method for converting a string to an `ObjectHash` object, consider the following: +/// 1. The input string `s` should be a pre-calculated hexadecimal string, either 40 characters in length for SHA1 or 64 characters for SHA256. +/// This string represents a hash and should conform to the standard hash format. /// 2. It is necessary to explicitly import the `FromStr` trait to utilize the `from_str` method. Include the import /// statement `use std::str::FromStr;` in your code before invoking the `from_str` function. This import ensures -/// that the `from_str` method is available for converting strings to `SHA1` objects. -impl std::str::FromStr for SHA1 { +impl FromStr for ObjectHash { type Err = String; fn from_str(s: &str) -> Result { - let mut h = SHA1::default(); - if s.len() != 40 { - return Err("The length of the string is not 40".to_string()); + match s.len() { + 40 => { + let mut h = [0u8; 20]; + let bytes = hex::decode(s).map_err(|e| e.to_string())?; + h.copy_from_slice(bytes.as_slice()); + Ok(ObjectHash::Sha1(h)) + } + 64 => { + let mut h = [0u8; 32]; + let bytes = hex::decode(s).map_err(|e| e.to_string())?; + h.copy_from_slice(bytes.as_slice()); + Ok(ObjectHash::Sha256(h)) + } + _ => Err("Invalid hash length".to_string()), } - let bytes = hex::decode(s).map_err(|e| e.to_string())?; - h.0.copy_from_slice(bytes.as_slice()); - Ok(h) } } -/// Implementation of the `SHA1` struct. -/// -/// The naming conventions for the methods in this implementation are designed to be intuitive and self-explanatory: -/// -/// 1. `new` Prefix: -/// Methods starting with `new` are used for computing an SHA-1 hash from given data, signifying the creation of -/// a new `SHA1` instance. For example, `pub fn new(data: &Vec) -> SHA1` takes a byte vector and calculates its SHA-1 hash. +/// Implementation of methods for the [`ObjectHash`] enum. +/// 1. The `kind` method determines the type of hash (SHA1 or SHA256) based on the variant of the `ObjectHash` enum. +/// 2. The `size` method returns the size of the hash in bytes, utilizing the `kind` method to determine the appropriate size. +/// 3. The `new` method computes the hash of the provided data using the specified hash kind (SHA1 or SHA256) and returns +/// an `ObjectHash` instance containing the computed hash. +/// 4. `from` Prefix:Methods to create an `ObjectHash` from different sources: +/// - `from_type_and_data`: Constructs an `ObjectHash` from an object type and its associated data. +/// - `from_bytes`: Creates an `ObjectHash` from a byte slice, ensuring the length matches the expected hash size. +/// - `from_stream`: Reads bytes from a stream to create an `ObjectHash`, ensuring the correct number of bytes are read based on the hash kind. +/// 5. `to` Prefix:Methods to convert an `ObjectHash` to different formats: +/// - `to_color_str`: Converts the hash to a colored string representation for display purposes +/// - `to_data`: Converts the hash to a byte vector. +/// - `_to_string`: Converts the hash to a hexadecimal string representation. /// -/// 2. `from` Prefix: -/// Methods beginning with `from` are intended for creating a `SHA1` instance from an existing, pre-calculated value. -/// This implies direct derivation of the `SHA1` object from the provided input. For instance, `pub fn from_bytes(bytes: &[u8]) -> SHA1` -/// constructs a `SHA1` from a 20-byte array representing an SHA-1 hash. -/// -/// 3. `to` Prefix: -/// Methods with the `to` prefix are used for outputting the `SHA1` value in various formats. This prefix indicates a transformation or -/// conversion of the `SHA1` instance into another representation. For example, `pub fn to_string(self) -> String` converts the SHA1 -/// value to a plain hexadecimal string, and `pub fn to_data(self) -> Vec` converts it into a byte vector. The `to` prefix -/// thus serves as a clear indicator that the method is exporting or transforming the SHA1 value into a different format. -/// -/// These method naming conventions (`new`, `from`, `to`) provide clarity and predictability in the API, making it easier for users -/// to understand the intended use and functionality of each method within the `SHA1` struct. -impl SHA1 { - // The size of the SHA-1 hash value in bytes - pub const SIZE: usize = 20; - - /// Calculate the SHA-1 hash of the byte slice, then create a Hash value - pub fn new(data: &[u8]) -> SHA1 { - let h = sha1::Sha1::digest(data); - SHA1::from_bytes(h.as_ref()) - } - /// Create a Hash from the object type and data - /// This function is used to create a SHA1 hash from the object type and data. - /// It constructs a byte vector that includes the object type, the size of the data, - /// and the data itself, and then computes the SHA1 hash of this byte vector. - /// - /// Hash compute <- {Object Type}+{ }+{Object Size(before compress)}+{\x00}+{Object Content(before compress)} - pub fn from_type_and_data(object_type: ObjectType, data: &[u8]) -> SHA1 { +impl ObjectHash { + /// returns the kind of hash + pub fn kind(&self) -> HashKind { + match self { + ObjectHash::Sha1(_) => HashKind::Sha1, + ObjectHash::Sha256(_) => HashKind::Sha256, + } + } + /// returns the size of hash in bytes + pub fn size(&self) -> usize { + self.kind().size() + } + + /// Calculates the hash of the given data using the specified hash kind. + pub fn new(data: &[u8]) -> ObjectHash { + match get_hash_kind() { + HashKind::Sha1 => { + let h = sha1::Sha1::digest(data); + let mut bytes = [0u8; 20]; + bytes.copy_from_slice(h.as_ref()); + ObjectHash::Sha1(bytes) + } + HashKind::Sha256 => { + let h = sha2::Sha256::digest(data); + let mut bytes = [0u8; 32]; + bytes.copy_from_slice(h.as_ref()); + ObjectHash::Sha256(bytes) + } + } + } + /// Create ObjectHash from object type and data + pub fn from_type_and_data(object_type: ObjectType, data: &[u8]) -> ObjectHash { let mut d: Vec = Vec::new(); d.extend(object_type.to_data().unwrap()); d.push(b' '); d.extend(data.len().to_string().as_bytes()); d.push(b'\x00'); d.extend(data); - SHA1::new(&d) + ObjectHash::new(&d) } + /// Create ObjectHash from a byte slice + pub fn from_bytes(bytes: &[u8]) -> Result { + let expected_len = get_hash_kind().size(); + if bytes.len() != expected_len { + return Err(format!( + "Invalid byte length: got {}, expected {}", + bytes.len(), + expected_len + )); + } - /// Create Hash from a byte array, which is a 20-byte array already calculated - pub fn from_bytes(bytes: &[u8]) -> SHA1 { - let mut h = SHA1::default(); - h.0.copy_from_slice(bytes); - h + match get_hash_kind() { + HashKind::Sha1 => { + let mut h = [0u8; 20]; + h.copy_from_slice(bytes); + Ok(ObjectHash::Sha1(h)) + } + HashKind::Sha256 => { + let mut h = [0u8; 32]; + h.copy_from_slice(bytes); + Ok(ObjectHash::Sha256(h)) + } + } } - - /// Read the Hash value from the stream - /// This function will read exactly 20 bytes from the stream - pub fn from_stream(data: &mut impl io::Read) -> io::Result { - let mut h = SHA1::default(); - data.read_exact(&mut h.0)?; - Ok(h) + /// Create ObjectHash from a stream + pub fn from_stream(data: &mut impl io::Read) -> io::Result { + match get_hash_kind() { + HashKind::Sha1 => { + let mut h = [0u8; 20]; + data.read_exact(&mut h)?; + Ok(ObjectHash::Sha1(h)) + } + HashKind::Sha256 => { + let mut h = [0u8; 32]; + data.read_exact(&mut h)?; + Ok(ObjectHash::Sha256(h)) + } + } } /// Export sha1 value to String with the color @@ -148,16 +231,39 @@ impl SHA1 { /// Export sha1 value to a byte array pub fn to_data(self) -> Vec { - self.0.to_vec() + self.as_ref().to_vec() } /// [`core::fmt::Display`] is somewhat expensive, /// use this hack to get a string more efficiently pub fn _to_string(&self) -> String { - hex::encode(self.0) + hex::encode(self.as_ref()) } + + /// Get mutable hash as byte slice + pub fn as_mut_bytes(&mut self) -> &mut [u8] { + match self { + ObjectHash::Sha1(bytes) => bytes.as_mut_slice(), + ObjectHash::Sha256(bytes) => bytes.as_mut_slice(), + } + } +} +thread_local! { + /// Thread-local variable to store the current hash kind. + /// This allows different threads to work with different hash algorithms concurrently + /// without interfering with each other. + static CURRENT_HASH_KIND: RefCell = RefCell::new(HashKind::default()); +} +pub fn set_hash_kind(kind: HashKind) { + CURRENT_HASH_KIND.with(|h| { + *h.borrow_mut() = kind; + }); } +/// Retrieves the hash kind for the current thread. +pub fn get_hash_kind() -> HashKind { + CURRENT_HASH_KIND.with(|h| *h.borrow()) +} #[cfg(test)] mod tests { @@ -168,24 +274,50 @@ mod tests { use std::str::FromStr; use std::{env, path::PathBuf}; - use crate::hash::SHA1; + use crate::hash::{HashKind, ObjectHash, get_hash_kind, set_hash_kind}; + // A guard to reset the hash kind after the test + struct HashKindGuard { + prev: HashKind, + } + impl Drop for HashKindGuard { + fn drop(&mut self) { + set_hash_kind(self.prev); + } + } + fn set_hash_kind_for_test(kind: HashKind) -> HashKindGuard { + let prev = get_hash_kind(); + set_hash_kind(kind); + HashKindGuard { prev } + } #[test] fn test_sha1_new() { + // Set hash kind to SHA1 for this test + let _guard = set_hash_kind_for_test(HashKind::Sha1); // Example input let data = "Hello, world!".as_bytes(); // Generate SHA1 hash from the input data - let sha1 = SHA1::new(data); + let sha1 = ObjectHash::new(data); // Known SHA1 hash for "Hello, world!" let expected_sha1_hash = "943a702d06f34599aee1f8da8ef9f7296031d699"; assert_eq!(sha1.to_string(), expected_sha1_hash); } + #[test] + fn test_sha256_new() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let data = "Hello, world!".as_bytes(); + let sha256 = ObjectHash::new(data); + let expected_sha256_hash = + "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3"; + assert_eq!(sha256.to_string(), expected_sha256_hash); + } #[test] fn test_signature_without_delta() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/pack-1d0e6c14760c956c173ede71cb28f33d921e232f.pack"); @@ -195,63 +327,144 @@ mod tests { buffered.seek(SeekFrom::End(-20)).unwrap(); let mut buffer = vec![0; 20]; buffered.read_exact(&mut buffer).unwrap(); - let signature = SHA1::from_bytes(buffer.as_ref()); + let signature = ObjectHash::from_bytes(buffer.as_ref()).unwrap(); assert_eq!( signature.to_string(), "1d0e6c14760c956c173ede71cb28f33d921e232f" ); } + #[test] + fn test_signature_without_delta_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let f = std::fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + + buffered.seek(SeekFrom::End(-32)).unwrap(); + let mut buffer = vec![0; 32]; + buffered.read_exact(&mut buffer).unwrap(); + let signature = ObjectHash::from_bytes(buffer.as_ref()).unwrap(); + assert_eq!( + signature.to_string(), + "78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d" + ); + } #[test] fn test_sha1_from_bytes() { - let sha1 = SHA1::from_bytes(&[ + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let sha1 = ObjectHash::from_bytes(&[ 0x8a, 0xb6, 0x86, 0xea, 0xfe, 0xb1, 0xf4, 0x47, 0x02, 0x73, 0x8c, 0x8b, 0x0f, 0x24, 0xf2, 0x56, 0x7c, 0x36, 0xda, 0x6d, - ]); + ]) + .unwrap(); assert_eq!(sha1.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } + #[test] + fn test_sha256_from_bytes() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + // Pre-calculated SHA256 hash for "abc" + let sha256 = ObjectHash::from_bytes(&[ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, + 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, + 0xf2, 0x00, 0x15, 0xad, + ]) + .unwrap(); + + assert_eq!( + sha256.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } #[test] fn test_from_stream() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let source = [ 0x8a, 0xb6, 0x86, 0xea, 0xfe, 0xb1, 0xf4, 0x47, 0x02, 0x73, 0x8c, 0x8b, 0x0f, 0x24, 0xf2, 0x56, 0x7c, 0x36, 0xda, 0x6d, ]; let mut reader = std::io::Cursor::new(source); - let sha1 = SHA1::from_stream(&mut reader).unwrap(); + let sha1 = ObjectHash::from_stream(&mut reader).unwrap(); assert_eq!(sha1.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } - + #[test] + fn test_sha256_from_stream() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let source = [ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, + 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, + 0xf2, 0x00, 0x15, 0xad, + ]; + let mut reader = std::io::Cursor::new(source); + let sha256 = ObjectHash::from_stream(&mut reader).unwrap(); + assert_eq!( + sha256.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } #[test] fn test_sha1_from_str() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; - match SHA1::from_str(hash_str) { + match ObjectHash::from_str(hash_str) { Ok(hash) => { assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } Err(e) => println!("Error: {e}"), } } + #[test] + fn test_sha256_from_str() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash_str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + match ObjectHash::from_str(hash_str) { + Ok(hash) => { + assert_eq!( + hash.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + Err(e) => println!("Error: {e}"), + } + } #[test] fn test_sha1_to_string() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; - match SHA1::from_str(hash_str) { + match ObjectHash::from_str(hash_str) { Ok(hash) => { assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } Err(e) => println!("Error: {e}"), } } - + #[test] + fn test_sha256_to_string() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash_str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + match ObjectHash::from_str(hash_str) { + Ok(hash) => { + assert_eq!( + hash.to_string(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + Err(e) => println!("Error: {e}"), + } + } #[test] fn test_sha1_to_data() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; - match SHA1::from_str(hash_str) { + match ObjectHash::from_str(hash_str) { Ok(hash) => { assert_eq!( hash.to_data(), @@ -264,4 +477,22 @@ mod tests { Err(e) => println!("Error: {e}"), } } + #[test] + fn test_sha256_to_data() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash_str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + match ObjectHash::from_str(hash_str) { + Ok(hash) => { + assert_eq!( + hash.to_data(), + vec![ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, + 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad, + ] + ); + } + Err(e) => println!("Error: {e}"), + } + } } diff --git a/tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack b/tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack new file mode 100644 index 00000000..62843b81 --- /dev/null +++ b/tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9287eaf658f377eeb9956a17555b214291398bdb36c40d0527180a6c707dcce +size 1039 From 775f1ec7288481b341f7e27e3e96ebdf1aadfe19 Mon Sep 17 00:00:00 2001 From: jackieismpc Date: Tue, 25 Nov 2025 21:19:18 +0800 Subject: [PATCH 2/7] 1.Temporarily fix all issues introduced by the hash abstraction 2.Ensure hash.rs passes all tests 3.Fix issues in the object module after the refactor and add new test cases for sha256 Signed-off-by: jackieismpc --- .gitignore | 2 + src/hash.rs | 34 +++--- src/internal/index.rs | 10 +- src/internal/object/blob.rs | 24 +++- src/internal/object/commit.rs | 109 ++++++++++++++---- src/internal/object/mod.rs | 10 +- src/internal/object/note.rs | 208 +++++++++++++++++++++++++++++----- src/internal/object/tag.rs | 19 ++-- src/internal/object/tree.rs | 162 +++++++++++++++++++++++--- src/internal/pack/cache.rs | 5 +- src/internal/pack/decode.rs | 4 +- src/internal/pack/encode.rs | 4 +- src/internal/pack/utils.rs | 2 +- src/internal/pack/wrapper.rs | 4 +- 14 files changed, 480 insertions(+), 117 deletions(-) diff --git a/.gitignore b/.gitignore index 26a8c694..25959e87 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ buck-out/ .vscode/ +src/.DS_Store +.DS_Store diff --git a/src/hash.rs b/src/hash.rs index 51612d6f..8389d9d3 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -10,7 +10,7 @@ use bincode::{Decode, Encode}; use colored::Colorize; use serde::{Deserialize, Serialize}; use sha1::Digest; - +pub type SHA1 = ObjectHash; /// The [`SHA1`] struct, encapsulating a `[u8; 20]` array, is specifically designed to represent Git hash IDs. /// In Git's context, these IDs are 40-character hexadecimal strings generated via the SHA-1 algorithm. /// Each Git object receives a unique hash ID based on its content, serving as an identifier for its location @@ -264,6 +264,22 @@ pub fn set_hash_kind(kind: HashKind) { pub fn get_hash_kind() -> HashKind { CURRENT_HASH_KIND.with(|h| *h.borrow()) } +/// A guard to reset the hash kind after the test +pub struct HashKindGuard { + prev: HashKind, +} +/// Implementation of the `Drop` trait for the `HashKindGuard` struct. +impl Drop for HashKindGuard { + fn drop(&mut self) { + set_hash_kind(self.prev); + } +} +/// Sets the hash kind for the current thread and returns a guard to reset it later. +pub fn set_hash_kind_for_test(kind: HashKind) -> HashKindGuard { + let prev = get_hash_kind(); + set_hash_kind(kind); + HashKindGuard { prev } +} #[cfg(test)] mod tests { @@ -274,21 +290,7 @@ mod tests { use std::str::FromStr; use std::{env, path::PathBuf}; - use crate::hash::{HashKind, ObjectHash, get_hash_kind, set_hash_kind}; - // A guard to reset the hash kind after the test - struct HashKindGuard { - prev: HashKind, - } - impl Drop for HashKindGuard { - fn drop(&mut self) { - set_hash_kind(self.prev); - } - } - fn set_hash_kind_for_test(kind: HashKind) -> HashKindGuard { - let prev = get_hash_kind(); - set_hash_kind(kind); - HashKindGuard { prev } - } + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; #[test] fn test_sha1_new() { diff --git a/src/internal/index.rs b/src/internal/index.rs index 00b9c2e5..5c94acf4 100644 --- a/src/internal/index.rs +++ b/src/internal/index.rs @@ -12,7 +12,7 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use sha1::{Digest, Sha1}; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::{SHA1, get_hash_kind}; use crate::internal::pack::wrapper::Wrapper; use crate::utils; @@ -287,7 +287,7 @@ impl Index { } // Extensions - while file.bytes_read() + SHA1::SIZE < total_size as usize { + while file.bytes_read() + get_hash_kind().size() < total_size as usize { // The remaining 20 bytes must be checksum let sign = utils::read_bytes(file, 4)?; println!( @@ -341,7 +341,7 @@ impl Index { entry_bytes.write_u32::(entry.uid)?; entry_bytes.write_u32::(entry.gid)?; entry_bytes.write_u32::(entry.size)?; - entry_bytes.write_all(&entry.hash.0)?; + entry_bytes.write_all(&entry.hash.as_ref())?; entry_bytes.write_u16::((&entry.flags).try_into().unwrap())?; entry_bytes.write_all(entry.name.as_bytes())?; let padding = 8 - ((22 + entry.name.len()) % 8); @@ -385,7 +385,7 @@ impl Index { let mut file = File::open(&abs_path)?; let mut hasher = Sha1::new(); io::copy(&mut file, &mut hasher)?; - let new_hash = SHA1::from_bytes(&hasher.finalize()); + let new_hash = SHA1::from_bytes(&hasher.finalize()).unwrap(); // refresh index if entry.ctime != new_ctime @@ -592,7 +592,7 @@ mod tests { source.push("Cargo.toml"); let file = Path::new(source.as_path()); // use as a normal file - let hash = SHA1::from_bytes(&[0; 20]); + let hash = SHA1::from_bytes(&[0; 20]).unwrap(); let workdir = Path::new("../"); let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); println!("{entry}"); diff --git a/src/internal/object/blob.rs b/src/internal/object/blob.rs index a2a2cc77..f846bcd0 100644 --- a/src/internal/object/blob.rs +++ b/src/internal/object/blob.rs @@ -17,7 +17,7 @@ //! $ echo "Hello, world!" | git hash-object -w --stdin //! ``` //! -//! This will output an SHA-1 hash, which is the ID of the newly created blob object. +//! This will output an SHA-1/ SHA-256 hash, which is the ID of the newly created blob object. //! The contents of the blob object would look something like this: //! //! ```bash @@ -30,14 +30,14 @@ use std::fmt::Display; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::types::ObjectType; /// **The Blob Object** #[derive(Eq, Debug, Clone)] pub struct Blob { - pub id: SHA1, + pub id: ObjectHash, pub data: Vec, } @@ -57,7 +57,7 @@ impl Display for Blob { impl ObjectTrait for Blob { /// Creates a new object from a byte slice. - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -94,8 +94,8 @@ impl Blob { /// - some file content can't be represented as a string (UTF-8), so we need to use bytes. pub fn from_content_bytes(content: Vec) -> Self { Blob { - // Calculate the SHA1 hash from the type and content - id: SHA1::from_type_and_data(ObjectType::Blob, &content), + // Calculate the hash from the type and content + id: ObjectHash::from_type_and_data(ObjectType::Blob, &content), data: content, } } @@ -104,8 +104,10 @@ impl Blob { #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; #[test] fn test_blob_from_content() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let content = "Hello, world!"; let blob = Blob::from_content(content); assert_eq!( @@ -113,4 +115,14 @@ mod tests { "5dd01c177f5d7d1be5346a5bc18a569a7410c2ef" ); } + #[test] + fn test_blob_from_content_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let content = "Hello, world!"; + let blob = Blob::from_content(content); + assert_eq!( + blob.id.to_string(), + "178b5fbed164aee269fee7323badf7269cca0eed0875717b0d2d4f9819164c3f" + ); + } } diff --git a/src/internal/object/commit.rs b/src/internal/object/commit.rs index 6112a001..679f65de 100644 --- a/src/internal/object/commit.rs +++ b/src/internal/object/commit.rs @@ -5,7 +5,7 @@ //! //! Each commit object in Git contains the following information: //! -//! - A unique SHA-1 hash that identifies the commit. +//! - A unique SHA-1/ SHA-256 hash that identifies the commit. //! - The author and committer of the commit (which may be different people). //! - The date and time the commit was made. //! - A commit message that describes the changes made in the commit. @@ -15,7 +15,7 @@ use std::fmt::Display; use std::str::FromStr; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; use crate::internal::object::signature::Signature; @@ -36,9 +36,9 @@ use serde::Serialize; /// - The message field contains the commit message, which maybe include signed or DCO. #[derive(Eq, Debug, Clone, Serialize, Deserialize, Decode, Encode)] pub struct Commit { - pub id: SHA1, - pub tree_id: SHA1, - pub parent_commit_ids: Vec, + pub id: ObjectHash, + pub tree_id: ObjectHash, + pub parent_commit_ids: Vec, pub author: Signature, pub committer: Signature, pub message: String, @@ -65,12 +65,12 @@ impl Commit { pub fn new( author: Signature, committer: Signature, - tree_id: SHA1, - parent_commit_ids: Vec, + tree_id: ObjectHash, + parent_commit_ids: Vec, message: &str, ) -> Commit { let mut commit = Commit { - id: SHA1::default(), + id: ObjectHash::default(), tree_id, parent_commit_ids, author, @@ -79,7 +79,7 @@ impl Commit { }; // Calculate the hash of the commit object // The hash is calculated from the type and data of the commit object - let hash = SHA1::from_type_and_data(ObjectType::Commit, &commit.to_data().unwrap()); + let hash = ObjectHash::from_type_and_data(ObjectType::Commit, &commit.to_data().unwrap()); commit.id = hash; commit } @@ -89,13 +89,17 @@ impl Commit { /// and a fixed email address. /// It also sets the commit message to the provided string. /// # Arguments - /// - `tree_id`: The SHA1 hash of the tree object that this commit points to. - /// - `parent_commit_ids`: A vector of SHA1 hashes of the parent commits. + /// - `tree_id`: The SHA1/ SHA-256 hash of the tree object that this commit points to. + /// - `parent_commit_ids`: A vector of SHA1/ SHA-256 hashes of the parent commits. /// - `message`: A string containing the commit message. /// # Returns /// A new `Commit` object with the specified tree ID, parent commit IDs, and commit message. /// The author and committer signatures are generated using the current time and a fixed email address. - pub fn from_tree_id(tree_id: SHA1, parent_commit_ids: Vec, message: &str) -> Commit { + pub fn from_tree_id( + tree_id: ObjectHash, + parent_commit_ids: Vec, + message: &str, + ) -> Commit { let author = Signature::from_data( format!( "author mega {} +0800", @@ -149,14 +153,14 @@ impl Commit { } impl ObjectTrait for Commit { - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { let mut commit = data; // Find the tree id and remove it from the data let tree_end = commit.find_byte(0x0a).unwrap(); - let tree_id: SHA1 = SHA1::from_str( + let tree_id: ObjectHash = ObjectHash::from_str( String::from_utf8(commit[5..tree_end].to_owned()) // 5 is the length of "tree " .unwrap() .as_str(), @@ -170,12 +174,12 @@ impl ObjectTrait for Commit { // Find all parent commit ids // The parent commit ids are all the lines that start with "parent " // We can use find_iter to find all occurrences of "parent " - // and then extract the SHA1 hashes from them. - let parent_commit_ids: Vec = commit[..author_begin] + // and then extract the SHA1/ SHA-256 hashes from them. + let parent_commit_ids: Vec = commit[..author_begin] .find_iter("parent") .map(|parent| { let parent_end = commit[parent..].find_byte(0x0a).unwrap(); - SHA1::from_str( + ObjectHash::from_str( // 7 is the length of "parent " String::from_utf8(commit[parent + 7..parent + parent_end].to_owned()) .unwrap() @@ -249,9 +253,11 @@ impl ObjectTrait for Commit { #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; use std::str::FromStr; fn basic_commit() -> Commit { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let raw_commit = br#"tree 341e54913a3a43069f2927cc0f703e5a9f730df1 author benjamin.747 1757467768 +0800 committer benjamin.747 1757491219 +0800 @@ -276,22 +282,52 @@ gpgsig -----BEGIN PGP SIGNATURE----- test parse commit from bytes "#; - let hash = SHA1::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap(); + let hash = ObjectHash::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap(); Commit::from_bytes(raw_commit, hash).unwrap() } + fn basic_commit_sha256() -> Commit { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let raw_commit = br#"tree 0250024cf99636335fff1070e4220c5d8f67cb8633572d54b304629ad5382760 +parent 33324c6819589e8eed81d6c72f216469151a0f2dbe7f42ba021d8b63049eb754 +author jackieismpc 1764061895 +0800 +committer jackieismpc 1764061895 +0800 +gpgsig-sha256 -----BEGIN PGP SIGNATURE----- + + iQIzBAABCAAdFiEEzW/BI6wDXimDk/4lItD7G/h4TUsFAmklcscACgkQItD7G/h4 + TUtKFRAAtJq9tdl9XdND1ef2dXVQYCkQQlSdNHe2AR/QRVOPI39ZjD5aajRmZoE2 + rKDenNML1ruiGEm+K3ntRDjus+3QF5Xkhj1D6eImQt6RXyOlo64I+GLRKlzw80Sl + hrd+l1eeuS4n46Z0U9fo1Qgc/crSn2VhUtLHJjvRntJoOb1vNreI2Y42Zmal3oVT + fQNQ7mqzh3KuWoa8T6nVrLaLH1vl9qhRgkPcIRbFf+ECbB96qykHqcbdHuneSgfx + +REpr1cedilkQlX81JrQ8Ntf4QFUPPHALl27/G6oPLT714cflEbvcFw7rNR+ktcD + ZJIMu5Cl7X3/v5e0od/hF9uPfiLHckUsOXiMFLfqRdZx/5XeQFWRpq4eYcW7e89e + 3wJoBA2lCk8SHTBfsprKMpAweXJF9FCjRT5f9Zse2grqH81aQeNJnpSOoCq86oc/ + nxhi8+rbIbClLCGQoGF7sE/fvmKqcex++JnXHcHTtK002Gnh3oHX07sbahlcGuYY + kg4QhXiLTQ5GfXnEnTPdFqbOVG02vEEsNeRgkmOz4c8Pm1FTDyOkuXd/Igvy7A9R + MZwQcJ6E4MnsMnoH8FKswGqCD7ftwtJtRzryORBVzvPKALufIXDVLyBbae9dxdej + bcpUK1bGtDljlwNtbLIOu+F1y2OVh7Tn3zxaQLcEhbUe2tP6rGk= + =nJMO + -----END PGP SIGNATURE----- + +signed sha256 commit for test"#; + let hash = ObjectHash::from_str( + "ed43b50437e260a4d8fedacbaa38bad28b54cc424925e4180d9f186afaa0508c", + ) + .unwrap(); + Commit::from_bytes(raw_commit.as_bytes(), hash).unwrap() + } #[test] fn test_from_bytes_with_gpgsig() { let commit = basic_commit(); assert_eq!( commit.id, - SHA1::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap() + ObjectHash::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap() ); assert_eq!( commit.tree_id, - SHA1::from_str("341e54913a3a43069f2927cc0f703e5a9f730df1").unwrap() + ObjectHash::from_str("341e54913a3a43069f2927cc0f703e5a9f730df1").unwrap() ); assert_eq!(commit.author.name, "benjamin.747"); @@ -304,10 +340,41 @@ test parse commit from bytes assert!(commit.message.contains("-----END PGP SIGNATURE-----")); assert!(commit.message.contains("test parse commit from bytes")); } - + #[test] + fn test_from_bytes_with_gpgsig_sha256() { + let commit = basic_commit_sha256(); + assert_eq!( + commit.id, + ObjectHash::from_str( + "ed43b50437e260a4d8fedacbaa38bad28b54cc424925e4180d9f186afaa0508c" + ) + .unwrap() + ); + assert_eq!( + commit.tree_id, + ObjectHash::from_str( + "0250024cf99636335fff1070e4220c5d8f67cb8633572d54b304629ad5382760" + ) + .unwrap() + ); + assert_eq!(commit.author.name, "jackieismpc"); + assert_eq!(commit.author.email, "jackieismpc@gmail.com"); + assert_eq!(commit.committer.name, "jackieismpc"); + // check message content(must contains gpgsig-sha256 and content) + assert!(commit.message.contains("-----BEGIN PGP SIGNATURE-----")); + assert!(commit.message.contains("-----END PGP SIGNATURE-----")); + assert!(commit.message.contains("signed sha256 commit for test")); + } #[test] fn test_format_message_with_pgp_signature() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let commit = basic_commit(); assert_eq!(commit.format_message(), "test parse commit from bytes"); } + #[test] + fn test_format_message_with_pgp_signature_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let commit = basic_commit_sha256(); + assert_eq!(commit.format_message(), "signed sha256 commit for test"); + } } diff --git a/src/internal/object/mod.rs b/src/internal/object/mod.rs index 1c1569a5..4478fa6f 100644 --- a/src/internal/object/mod.rs +++ b/src/internal/object/mod.rs @@ -17,11 +17,11 @@ use sha1::Digest; use crate::internal::object::types::ObjectType; use crate::internal::zlib::stream::inflate::ReadBoxed; -use crate::{errors::GitError, hash::SHA1}; +use crate::{errors::GitError, hash::ObjectHash}; pub trait ObjectTrait: Send + Sync + Display { /// Creates a new object from a byte slice. - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized; @@ -36,7 +36,11 @@ pub trait ObjectTrait: Send + Sync + Display { read.read_to_end(&mut content).unwrap(); let h = read.hash.clone(); let hash_str = h.finalize(); - Self::from_bytes(&content, SHA1::from_str(&format!("{hash_str:x}")).unwrap()).unwrap() + Self::from_bytes( + &content, + ObjectHash::from_str(&format!("{hash_str:x}")).unwrap(), + ) + .unwrap() } /// Returns the type of the object. diff --git a/src/internal/object/note.rs b/src/internal/object/note.rs index ab39e409..dd398eed 100644 --- a/src/internal/object/note.rs +++ b/src/internal/object/note.rs @@ -16,7 +16,7 @@ use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; @@ -27,10 +27,10 @@ use crate::internal::object::ObjectType; /// association managed through Git's reference system. #[derive(Eq, Debug, Clone, Serialize, Deserialize, Decode, Encode)] pub struct Note { - /// The SHA-1 hash of this Note object (same as the underlying Blob) - pub id: SHA1, - /// The SHA-1 hash of the object this Note annotates (usually a commit) - pub target_object_id: SHA1, + /// The ObjectHash of this Note object (same as the underlying Blob) + pub id: ObjectHash, + /// The ObjectHash of the object this Note annotates (usually a commit) + pub target_object_id: ObjectHash, /// The textual content of the Note pub content: String, } @@ -53,15 +53,15 @@ impl Note { /// Create a new Note for the specified target object with the given content /// /// # Arguments - /// * `target_object_id` - The SHA-1 hash of the object to annotate + /// * `target_object_id` - The ObjectHash of the object to annotate /// * `content` - The textual content of the note /// /// # Returns /// A new Note instance with calculated ID based on the content - pub fn new(target_object_id: SHA1, content: String) -> Self { - // Calculate the SHA-1 hash for this Note's content + pub fn new(target_object_id: ObjectHash, content: String) -> Self { + // Calculate the SHA-1/ SHA-256 hash for this Note's content // Notes are stored as Blob objects in Git - let id = SHA1::from_type_and_data(ObjectType::Blob, content.as_bytes()); + let id = ObjectHash::from_type_and_data(ObjectType::Blob, content.as_bytes()); Self { id, @@ -81,7 +81,7 @@ impl Note { /// # Returns /// A new Note instance with default target object ID pub fn from_content(content: &str) -> Self { - Self::new(SHA1::default(), content.to_string()) + Self::new(ObjectHash::default(), content.to_string()) } /// Get the size of the Note content in bytes @@ -100,8 +100,8 @@ impl Note { /// without changing the Note's content or ID. /// /// # Arguments - /// * `new_target` - The new target object SHA-1 hash - pub fn set_target(&mut self, new_target: SHA1) { + /// * `new_target` - The new target object SHA-1/ SHA-256 hash + pub fn set_target(&mut self, new_target: ObjectHash) { self.target_object_id = new_target; } @@ -112,15 +112,15 @@ impl Note { /// /// # Arguments /// * `data` - The raw byte data (UTF-8 encoded text content) - /// * `hash` - The SHA-1 hash of this Note object - /// * `target_object_id` - The SHA-1 hash of the object this Note annotates + /// * `hash` - The SHA-1/ SHA-256 hash of this Note object + /// * `target_object_id` - The SHA-1/ SHA-256 hash of the object this Note annotates /// /// # Returns /// A Result containing the Note object with complete association info pub fn from_bytes_with_target( data: &[u8], - hash: SHA1, - target_object_id: SHA1, + hash: ObjectHash, + target_object_id: ObjectHash, ) -> Result { let content = String::from_utf8(data.to_vec()) .map_err(|e| GitError::InvalidNoteObject(format!("Invalid UTF-8 content: {}", e)))?; @@ -139,7 +139,7 @@ impl Note { /// /// # Returns /// A tuple of (object_data, target_object_id) - pub fn to_data_with_target(&self) -> Result<(Vec, SHA1), GitError> { + pub fn to_data_with_target(&self) -> Result<(Vec, ObjectHash), GitError> { let data = self.to_data()?; Ok((data, self.target_object_id)) } @@ -150,11 +150,11 @@ impl ObjectTrait for Note { /// /// # Arguments /// * `data` - The raw byte data (UTF-8 encoded text content) - /// * `hash` - The SHA-1 hash of this Note object + /// * `hash` - The SHA-1/ SHA-256 hash of this Note object /// /// # Returns /// A Result containing the Note object or an error - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -164,7 +164,7 @@ impl ObjectTrait for Note { Ok(Note { id: hash, - target_object_id: SHA1::default(), // Target association managed externally + target_object_id: ObjectHash::default(), // Target association managed externally content, }) } @@ -193,17 +193,39 @@ impl ObjectTrait for Note { #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; use std::str::FromStr; #[test] fn test_note_creation_and_serialization() { - let target_id = SHA1::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let target_id = ObjectHash::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); let content = "This commit needs review".to_string(); let note = Note::new(target_id, content.clone()); assert_eq!(note.target_object_id, target_id); assert_eq!(note.content, content); - assert_ne!(note.id, SHA1::default()); + assert_ne!(note.id, ObjectHash::default()); + assert_eq!(note.get_type(), ObjectType::Blob); + + // Test serialization + let data = note.to_data().unwrap(); + assert_eq!(data, content.as_bytes()); + assert_eq!(note.get_size(), content.len()); + } + #[test] + fn test_note_creation_and_serialization_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let target_id = ObjectHash::from_str( + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ) + .unwrap(); + let content = "This commit needs review".to_string(); + let note = Note::new(target_id, content.clone()); + + assert_eq!(note.target_object_id, target_id); + assert_eq!(note.content, content); + assert_ne!(note.id, ObjectHash::default()); assert_eq!(note.get_type(), ObjectType::Blob); // Test serialization @@ -214,15 +236,42 @@ mod tests { #[test] fn test_note_deserialization() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let content = "Deserialization test content"; + let hash = ObjectHash::from_str("fedcba0987654321fedcba0987654321fedcba09").unwrap(); + let target_id = ObjectHash::from_str("abcdef1234567890abcdef1234567890abcdef12").unwrap(); + + // Test basic deserialization + let note = Note::from_bytes(content.as_bytes(), hash).unwrap(); + assert_eq!(note.content, content); + assert_eq!(note.id, hash); + assert_eq!(note.target_object_id, ObjectHash::default()); + + // Test deserialization with target + let note_with_target = + Note::from_bytes_with_target(content.as_bytes(), hash, target_id).unwrap(); + assert_eq!(note_with_target.content, content); + assert_eq!(note_with_target.id, hash); + assert_eq!(note_with_target.target_object_id, target_id); + } + #[test] + fn test_note_deserialization_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let content = "Deserialization test content"; - let hash = SHA1::from_str("fedcba0987654321fedcba0987654321fedcba09").unwrap(); - let target_id = SHA1::from_str("abcdef1234567890abcdef1234567890abcdef12").unwrap(); + let hash = ObjectHash::from_str( + "fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321", + ) + .unwrap(); + let target_id = ObjectHash::from_str( + "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + ) + .unwrap(); // Test basic deserialization let note = Note::from_bytes(content.as_bytes(), hash).unwrap(); assert_eq!(note.content, content); assert_eq!(note.id, hash); - assert_eq!(note.target_object_id, SHA1::default()); + assert_eq!(note.target_object_id, ObjectHash::default()); // Test deserialization with target let note_with_target = @@ -234,7 +283,29 @@ mod tests { #[test] fn test_note_with_target_methods() { - let target_id = SHA1::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); + let _guard = set_hash_kind_for_test(HashKind::Sha1); + let target_id = ObjectHash::from_str("1234567890abcdef1234567890abcdef12345678").unwrap(); + let content = "Test note with target methods"; + let note = Note::new(target_id, content.to_string()); + + // Test serialization with target + let (data, returned_target) = note.to_data_with_target().unwrap(); + assert_eq!(data, content.as_bytes()); + assert_eq!(returned_target, target_id); + + // Test deserialization with target + let restored_note = Note::from_bytes_with_target(&data, note.id, target_id).unwrap(); + assert_eq!(restored_note, note); + assert_eq!(restored_note.target_object_id, target_id); + assert_eq!(restored_note.content, content); + } + #[test] + fn test_note_with_target_methods_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let target_id = ObjectHash::from_str( + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ) + .unwrap(); let content = "Test note with target methods"; let note = Note::new(target_id, content.to_string()); @@ -252,26 +323,103 @@ mod tests { #[test] fn test_note_error_handling() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); // Test invalid UTF-8 let invalid_utf8 = vec![0xFF, 0xFE, 0xFD]; - let hash = SHA1::from_str("3333333333333333333333333333333333333333").unwrap(); - let target = SHA1::from_str("4444444444444444444444444444444444444444").unwrap(); - + let hash = ObjectHash::from_str("3333333333333333333333333333333333333333").unwrap(); + let target = ObjectHash::from_str("4444444444444444444444444444444444444444").unwrap(); let result = Note::from_bytes(&invalid_utf8, hash); assert!(result.is_err()); let result_with_target = Note::from_bytes_with_target(&invalid_utf8, hash, target); assert!(result_with_target.is_err()); } + #[test] + fn test_note_error_handling_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + // Test invalid UTF-8 + let invalid_utf8 = vec![0xFF, 0xFE, 0xFD]; + let hash = ObjectHash::from_str( + "3333333333333333333333333333333333333333333333333333333333333333", + ) + .unwrap(); + let target = ObjectHash::from_str( + "4444444444444444444444444444444444444444444444444444444444444444", + ) + .unwrap(); + let result = Note::from_bytes(&invalid_utf8, hash); + assert!(result.is_err()); + let result_with_target = Note::from_bytes_with_target(&invalid_utf8, hash, target); + assert!(result_with_target.is_err()); + } #[test] fn test_note_demo_functionality() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); + // This is a demonstration test that shows the complete functionality + // It's kept separate from unit tests for clarity + println!("\n🚀 Git Note Object Demo - Best Practices"); + println!("=========================================="); + + let commit_id = ObjectHash::from_str("a1b2c3d4e5f6789012345678901234567890abcd").unwrap(); + + println!("\n1️⃣ Creating a new Note object:"); + let note = Note::new( + commit_id, + "Code review: LGTM! Great implementation.".to_string(), + ); + println!(" Target Commit: {}", note.target_object_id); + println!(" Note ID: {}", note.id); + println!(" Content: {}", note.content); + println!(" Size: {} bytes", note.get_size()); + + println!("\n2️⃣ Serializing Note with target association:"); + let (serialized_data, target_id) = note.to_data_with_target().unwrap(); + println!(" Serialized size: {} bytes", serialized_data.len()); + println!(" Target object ID: {}", target_id); + println!( + " Git object format: blob {}\\0", + note.content.len() + ); + println!( + " Raw data preview: {:?}...", + &serialized_data[..std::cmp::min(30, serialized_data.len())] + ); + + println!("\n3️⃣ Basic deserialization (ObjectTrait):"); + let basic_note = Note::from_bytes(&serialized_data, note.id).unwrap(); + println!(" Successfully deserialized!"); + println!( + " Target Commit: {} (default - target managed externally)", + basic_note.target_object_id + ); + println!(" Content: {}", basic_note.content); + println!(" Content matches: {}", note.content == basic_note.content); + + println!("\n4️⃣ Best practice deserialization (with target):"); + let complete_note = + Note::from_bytes_with_target(&serialized_data, note.id, target_id).unwrap(); + println!(" Successfully deserialized with target!"); + println!(" Target Commit: {}", complete_note.target_object_id); + println!(" Content: {}", complete_note.content); + println!(" Complete objects are equal: {}", note == complete_note); + + // Basic assertions to ensure demo works + assert_eq!(note, complete_note); + assert_eq!(target_id, commit_id); + } + #[test] + fn test_note_demo_functionality_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); // This is a demonstration test that shows the complete functionality // It's kept separate from unit tests for clarity println!("\n🚀 Git Note Object Demo - Best Practices"); println!("=========================================="); - let commit_id = SHA1::from_str("a1b2c3d4e5f6789012345678901234567890abcd").unwrap(); + let commit_id = ObjectHash::from_str( + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ) + .unwrap(); println!("\n1️⃣ Creating a new Note object:"); let note = Note::new( diff --git a/src/internal/object/tag.rs b/src/internal/object/tag.rs index bd984281..986e3e49 100644 --- a/src/internal/object/tag.rs +++ b/src/internal/object/tag.rs @@ -42,7 +42,7 @@ use std::str::FromStr; use bstr::ByteSlice; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; use crate::internal::object::signature::Signature; @@ -50,8 +50,8 @@ use crate::internal::object::signature::Signature; /// The tag object is used to Annotated tag #[derive(Eq, Debug, Clone)] pub struct Tag { - pub id: SHA1, - pub object_hash: SHA1, + pub id: ObjectHash, + pub object_hash: ObjectHash, pub object_type: ObjectType, pub tag_name: String, pub tagger: Signature, @@ -85,7 +85,7 @@ impl Tag { // } pub fn new( - object_hash: SHA1, + object_hash: ObjectHash, object_type: ObjectType, tag_name: String, tagger: Signature, @@ -96,7 +96,7 @@ impl Tag { "object {}\ntype {}\ntag {}\ntagger {}\n\n{}", object_hash, object_type, tag_name, tagger, message ); - let id = SHA1::from_type_and_data(ObjectType::Tag, data.as_bytes()); + let id = ObjectHash::from_type_and_data(ObjectType::Tag, data.as_bytes()); Self { id, @@ -119,7 +119,7 @@ impl ObjectTrait for Tag { /// tagger 0x0a # The name, email address, and date of the person who created the annotated tag /// /// ``` - fn from_bytes(row_data: &[u8], hash: SHA1) -> Result + fn from_bytes(row_data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -131,7 +131,7 @@ impl ObjectTrait for Tag { headers = &headers[..pos]; } - let mut object_hash: Option = None; + let mut object_hash: Option = None; let mut object_type: Option = None; let mut tag_name: Option = None; let mut tagger: Option = None; @@ -141,7 +141,7 @@ impl ObjectTrait for Tag { let hash_str = s.to_str().map_err(|_| { GitError::InvalidTagObject("Invalid UTF-8 in object hash".to_string()) })?; - object_hash = Some(SHA1::from_str(hash_str).map_err(|_| { + object_hash = Some(ObjectHash::from_str(hash_str).map_err(|_| { GitError::InvalidTagObject("Invalid object hash format".to_string()) })?); } else if let Some(s) = line.strip_prefix(b"type ") { @@ -189,12 +189,13 @@ impl ObjectTrait for Tag { /// /// ```bash - /// object 0x0a # The SHA-1 hash of the object that the annotated tag is attached to (usually a commit) + /// object 0x0a # The SHA-1/ SHA-256 hash of the object that the annotated tag is attached to (usually a commit) /// type 0x0a #The type of Git object that the annotated tag is attached to (usually 'commit') /// tag 0x0a # The name of the annotated tag(in UTF-8 encoding) /// tagger 0x0a # The name, email address, and date of the person who created the annotated tag /// /// ``` + /// When using SHA-1, `` is 40 hex chars; when using SHA-256, it is 64 hex chars. fn to_data(&self) -> Result, GitError> { let mut data = Vec::new(); diff --git a/src/internal/object/tree.rs b/src/internal/object/tree.rs index 5bc7a74c..13204c79 100644 --- a/src/internal/object/tree.rs +++ b/src/internal/object/tree.rs @@ -15,7 +15,7 @@ //! operations like merging and rebasing more quickly and accurately. //! use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::{ObjectHash, get_hash_kind}; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; use bincode::{Decode, Encode}; @@ -133,7 +133,7 @@ impl TreeItemMode { #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Hash, Encode, Decode)] pub struct TreeItem { pub mode: TreeItemMode, - pub id: SHA1, + pub id: ObjectHash, pub name: String, } @@ -151,7 +151,7 @@ impl Display for TreeItem { impl TreeItem { // Create a new TreeItem from a mode, id and name - pub fn new(mode: TreeItemMode, id: SHA1, name: String) -> Self { + pub fn new(mode: TreeItemMode, id: ObjectHash, name: String) -> Self { TreeItem { mode, id, name } } @@ -183,7 +183,7 @@ impl TreeItem { }; Ok(TreeItem { mode: TreeItemMode::tree_item_type_from_bytes(mode)?, - id: SHA1::from_bytes(id), + id: ObjectHash::from_bytes(id).unwrap(), name, }) } @@ -227,7 +227,7 @@ impl TreeItem { /// for each file or directory in the tree. #[derive(Eq, Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct Tree { - pub id: SHA1, + pub id: ObjectHash, pub tree_items: Vec, } @@ -262,7 +262,7 @@ impl Tree { } Ok(Tree { - id: SHA1::from_type_and_data(ObjectType::Tree, &data), + id: ObjectHash::from_type_and_data(ObjectType::Tree, &data), tree_items, }) } @@ -273,19 +273,19 @@ impl Tree { for item in &self.tree_items { data.extend_from_slice(item.to_data().as_slice()); } - self.id = SHA1::from_type_and_data(ObjectType::Tree, &data); + self.id = ObjectHash::from_type_and_data(ObjectType::Tree, &data); } } impl TryFrom<&[u8]> for Tree { type Error = GitError; fn try_from(data: &[u8]) -> Result { - let h = SHA1::from_type_and_data(ObjectType::Tree, data); + let h = ObjectHash::from_type_and_data(ObjectType::Tree, data); Tree::from_bytes(data, h) } } impl ObjectTrait for Tree { - fn from_bytes(data: &[u8], hash: SHA1) -> Result + fn from_bytes(data: &[u8], hash: ObjectHash) -> Result where Self: Sized, { @@ -295,8 +295,10 @@ impl ObjectTrait for Tree { // Find the position of the null byte (0x00) if let Some(index) = memchr::memchr(0x00, &data[i..]) { // Calculate the next position - let next = i + index + 21; - + let next = i + index + get_hash_kind().size() + 1; // +1 for the null byte + if next > data.len() { + return Err(GitError::InvalidTreeObject); + } //check bounds TreeItem::from_bytes will panic if out of bounds // Extract the bytes and create a TreeItem let item_data = &data[i..next]; let tree_item = TreeItem::from_bytes(item_data)?; @@ -339,16 +341,16 @@ impl ObjectTrait for Tree { #[cfg(test)] mod tests { - use std::str::FromStr; - - use crate::hash::SHA1; + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; use crate::internal::object::tree::{Tree, TreeItem, TreeItemMode}; + use std::str::FromStr; #[test] fn test_tree_item_new() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let tree_item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + ObjectHash::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), "hello-world".to_string(), ); @@ -358,12 +360,32 @@ mod tests { "8ab686eafeb1f44702738c8b0f24f2567c36da6d" ); } + #[test] + fn test_tree_item_new_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let tree_item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "hello-world".to_string(), + ); + + assert_eq!(tree_item.mode, TreeItemMode::Blob); + assert_eq!( + tree_item.id.to_string(), + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4" + ); + assert_eq!(tree_item.name, "hello-world"); + } #[test] fn test_tree_item_to_bytes() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let tree_item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + ObjectHash::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), "hello-world".to_string(), ); @@ -377,12 +399,46 @@ mod tests { ] ); } + #[test] + fn test_tree_item_to_bytes_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let tree_item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "hello-world".to_string(), + ); + + let bytes = tree_item.to_data(); + + // 一个小 helper:把十六进制字符串转成字节序列 + fn hex_to_bytes(s: &str) -> Vec { + assert!(s.len() % 2 == 0); + let mut out = Vec::with_capacity(s.len() / 2); + for i in (0..s.len()).step_by(2) { + let byte = u8::from_str_radix(&s[i..i + 2], 16).unwrap(); + out.push(byte); + } + out + } + + // 期望的编码:`"100644 hello-world\0" + <32字节的blob哈希>` + let mut expected = b"100644 hello-world\0".to_vec(); + expected.extend_from_slice(&hex_to_bytes( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + )); + + assert_eq!(bytes, expected); + } #[test] fn test_tree_item_from_bytes() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + ObjectHash::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), "hello-world".to_string(), ); @@ -392,12 +448,30 @@ mod tests { assert_eq!(tree_item.mode, TreeItemMode::Blob); assert_eq!(tree_item.id.to_string(), item.id.to_string()); } + #[test] + fn test_tree_item_from_bytes_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "1111111111111111111111111111111111111111111111111111111111111111", + ) + .unwrap(), + "hello-world".to_string(), + ); + let bytes = item.to_data(); + let tree_item = TreeItem::from_bytes(bytes.as_slice()).unwrap(); + + assert_eq!(tree_item.mode, TreeItemMode::Blob); + assert_eq!(tree_item.id.to_string(), item.id.to_string()); + } #[test] fn test_from_tree_items() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let item = TreeItem::new( TreeItemMode::Blob, - SHA1::from_str("17288789afffb273c8c394bc65e87d899b92897b").unwrap(), + ObjectHash::from_str("17288789afffb273c8c394bc65e87d899b92897b").unwrap(), "hello-world".to_string(), ); let tree = Tree::from_tree_items(vec![item]).unwrap(); @@ -407,4 +481,56 @@ mod tests { tree.id.to_string() ); } + #[test] + fn test_from_tree_items_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let items = vec![ + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "a.txt".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "fc2593998f8e1dec9c3a8be11557888134dad90ef5c7a2d6236ed75534c7698e", + ) + .unwrap(), + "b.txt".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "21513dcb4d6f9eb247db3b4c52158395d94f809cbaa2630bd2a7a474d9b39fab", + ) + .unwrap(), + "c.txt".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", + ) + .unwrap(), + "hello-world".to_string(), + ), + TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str( + "9ba9ae56288652bf32f074f922e37d3e95df8920b3cdfc053309595b8f86cbc6", + ) + .unwrap(), + "message.txt".to_string(), + ), + ]; + let tree = Tree::from_tree_items(items).unwrap(); + println!("{}", tree.id); + assert_eq!( + "d712a36aadfb47cabc7aaa90cf9e515773ba3bfc1fe3783730b387ce15c49261", + tree.id.to_string() + ); + } } diff --git a/src/internal/pack/cache.rs b/src/internal/pack/cache.rs index fa9a0698..4119ad52 100644 --- a/src/internal/pack/cache.rs +++ b/src/internal/pack/cache.rs @@ -10,7 +10,7 @@ use dashmap::{DashMap, DashSet}; use lru_mem::LruCache; use threadpool::ThreadPool; -use crate::hash::SHA1; +use crate::hash::{SHA1, get_hash_kind}; use crate::internal::pack::cache_object::{ ArcWrapper, CacheObject, FileLoadStore, MemSizeRecorder, }; @@ -90,7 +90,8 @@ impl Caches { /// generate the temp file path, hex string of the hash fn generate_temp_path(&self, tmp_path: &Path, hash: SHA1) -> PathBuf { // This is enough for the original path, 2 chars directory, 40 chars hash, and extra slashes - let mut path = PathBuf::with_capacity(self.tmp_path.capacity() + SHA1::SIZE * 2 + 5); + let mut path = + PathBuf::with_capacity(self.tmp_path.capacity() + get_hash_kind().size() * 2 + 5); path.push(tmp_path); let hash_str = hash._to_string(); path.push(&hash_str[..2]); // use first 2 chars as the directory diff --git a/src/internal/pack/decode.rs b/src/internal/pack/decode.rs index 1cd5ef47..6c1f920f 100644 --- a/src/internal/pack/decode.rs +++ b/src/internal/pack/decode.rs @@ -14,7 +14,7 @@ use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; use crate::errors::GitError; -use crate::hash::SHA1; +use crate::hash::{SHA1, get_hash_kind}; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::zstdelta; @@ -312,7 +312,7 @@ impl Pack { // Read 20 bytes to get the reference object SHA1 hash let ref_sha1 = SHA1::from_stream(pack).unwrap(); // Offset is incremented by 20 bytes - *offset += SHA1::SIZE; + *offset += get_hash_kind().size(); let (data, raw_size) = Pack::decompress_data(pack, size)?; *offset += raw_size; diff --git a/src/internal/pack/encode.rs b/src/internal/pack/encode.rs index 0f189977..f676af90 100644 --- a/src/internal/pack/encode.rs +++ b/src/internal/pack/encode.rs @@ -341,7 +341,7 @@ impl PackEncoder { // Hash signature let hash_result = self.inner_hash.clone().finalize(); - self.final_hash = Some(SHA1::from_bytes(&hash_result)); + self.final_hash = Some(SHA1::from_bytes(&hash_result).unwrap()); self.send_data(hash_result.to_vec()).await; self.drop_sender(); @@ -535,7 +535,7 @@ impl PackEncoder { // hash signature let hash_result = self.inner_hash.clone().finalize(); - self.final_hash = Some(SHA1::from_bytes(&hash_result)); + self.final_hash = Some(SHA1::from_bytes(&hash_result).unwrap()); self.send_data(hash_result.to_vec()).await; self.drop_sender(); Ok(()) diff --git a/src/internal/pack/utils.rs b/src/internal/pack/utils.rs index d98db98a..bb32da3f 100644 --- a/src/internal/pack/utils.rs +++ b/src/internal/pack/utils.rs @@ -270,7 +270,7 @@ pub fn calculate_object_hash(obj_type: ObjectType, data: &Vec) -> SHA1 { hash.update(data); let re: [u8; 20] = hash.finalize().into(); - SHA1(re) + SHA1::from_bytes(&re).unwrap() } /// Create an empty directory or clear the existing directory. pub fn create_empty_dir>(path: P) -> io::Result<()> { diff --git a/src/internal/pack/wrapper.rs b/src/internal/pack/wrapper.rs index 057d77a0..818d63ab 100644 --- a/src/internal/pack/wrapper.rs +++ b/src/internal/pack/wrapper.rs @@ -44,7 +44,7 @@ where /// This is a clone of the internal hash state finalized into a SHA1 hash. pub fn final_hash(&self) -> SHA1 { let re: [u8; 20] = self.hash.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes - SHA1(re) + SHA1::from_bytes(&re).unwrap() } } @@ -126,7 +126,7 @@ mod tests { hasher.update(data); let expected_hash: [u8; 20] = hasher.finalize().into(); - assert_eq!(hash_result.0, expected_hash); + assert_eq!(hash_result.as_ref(), expected_hash); Ok(()) } } From 69638ba587119b052794da2a5c39bfd0076bd70b Mon Sep 17 00:00:00 2001 From: jackieismpc Date: Wed, 26 Nov 2025 17:10:56 +0800 Subject: [PATCH 3/7] =?UTF-8?q?1.Completed=20pack=20module=20updates=20and?= =?UTF-8?q?=20added=20SHA-256=20tests=202.Introduced=20mocked=20pack=20fil?= =?UTF-8?q?es=20for=20testing=20purposes=EF=BC=88no=20big=20pack=20for=20s?= =?UTF-8?q?ha256=20test=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jackieismpc --- src/internal/pack/cache.rs | 146 ++++++- src/internal/pack/cache_object.rs | 138 ++++++- src/internal/pack/decode.rs | 192 ++++++++- src/internal/pack/encode.rs | 386 +++++++++++++++++- src/internal/pack/entry.rs | 4 +- src/internal/pack/mod.rs | 4 +- src/internal/pack/utils.rs | 56 ++- src/internal/pack/waitlist.rs | 8 +- src/internal/pack/wrapper.rs | 85 +++- ...6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack | 3 + ...ec752e00ac72798859d850ab1dcfd801beedd.pack | 3 + ...cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack | 3 + 12 files changed, 928 insertions(+), 100 deletions(-) create mode 100644 tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack create mode 100644 tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack create mode 100644 tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack diff --git a/src/internal/pack/cache.rs b/src/internal/pack/cache.rs index 4119ad52..08309112 100644 --- a/src/internal/pack/cache.rs +++ b/src/internal/pack/cache.rs @@ -10,7 +10,7 @@ use dashmap::{DashMap, DashSet}; use lru_mem::LruCache; use threadpool::ThreadPool; -use crate::hash::{SHA1, get_hash_kind}; +use crate::hash::ObjectHash; use crate::internal::pack::cache_object::{ ArcWrapper, CacheObject, FileLoadStore, MemSizeRecorder, }; @@ -20,29 +20,29 @@ pub trait _Cache { fn new(mem_size: Option, tmp_path: PathBuf, thread_num: usize) -> Self where Self: Sized; - fn get_hash(&self, offset: usize) -> Option; - fn insert(&self, offset: usize, hash: SHA1, obj: CacheObject) -> Arc; + fn get_hash(&self, offset: usize) -> Option; + fn insert(&self, offset: usize, hash: ObjectHash, obj: CacheObject) -> Arc; fn get_by_offset(&self, offset: usize) -> Option>; - fn get_by_hash(&self, h: SHA1) -> Option>; + fn get_by_hash(&self, h: ObjectHash) -> Option>; fn total_inserted(&self) -> usize; fn memory_used(&self) -> usize; fn clear(&self); } -impl lru_mem::HeapSize for SHA1 { +impl lru_mem::HeapSize for ObjectHash { fn heap_size(&self) -> usize { 0 } } pub struct Caches { - map_offset: DashMap, // offset to hash - hash_set: DashSet, // item in the cache + map_offset: DashMap, // offset to hash + hash_set: DashSet, // item in the cache // dropping large lru cache will take a long time on Windows without multi-thread IO // because "multi-thread IO" clone Arc, so it won't be dropped in the main thread, // and `CacheObjects` will be killed by OS after Process ends abnormally // Solution: use `mimalloc` - lru_cache: Mutex>>, + lru_cache: Mutex>>, mem_size: Option, tmp_path: PathBuf, path_prefixes: [Once; 256], @@ -52,14 +52,14 @@ pub struct Caches { impl Caches { /// only get object from memory, not from tmp file - fn try_get(&self, hash: SHA1) -> Option> { + fn try_get(&self, hash: ObjectHash) -> Option> { let mut map = self.lru_cache.lock().unwrap(); map.get(&hash).map(|x| x.data.clone()) } /// !IMPORTANT: because of the process of pack, the file must be written / be writing before, so it won't be dead lock /// fall back to temp to get item. **invoker should ensure the hash is in the cache, or it will block forever** - fn get_fallback(&self, hash: SHA1) -> io::Result> { + fn get_fallback(&self, hash: ObjectHash) -> io::Result> { let path = self.generate_temp_path(&self.tmp_path, hash); // read from tmp file let obj = { @@ -88,10 +88,10 @@ impl Caches { } /// generate the temp file path, hex string of the hash - fn generate_temp_path(&self, tmp_path: &Path, hash: SHA1) -> PathBuf { - // This is enough for the original path, 2 chars directory, 40 chars hash, and extra slashes + fn generate_temp_path(&self, tmp_path: &Path, hash: ObjectHash) -> PathBuf { + // This is enough for the original path, 2 chars directory, 40/64 chars hash, and extra slashes let mut path = - PathBuf::with_capacity(self.tmp_path.capacity() + get_hash_kind().size() * 2 + 5); + PathBuf::with_capacity(self.tmp_path.capacity() + hash.to_string().len() + 5); path.push(tmp_path); let hash_str = hash._to_string(); path.push(&hash_str[..2]); // use first 2 chars as the directory @@ -119,8 +119,9 @@ impl Caches { /// memory used by the index (exclude lru_cache which is contained in CacheObject::get_mem_size()) pub fn memory_used_index(&self) -> usize { - self.map_offset.capacity() * (std::mem::size_of::() + std::mem::size_of::()) - + self.hash_set.capacity() * (std::mem::size_of::()) + self.map_offset.capacity() + * (std::mem::size_of::() + std::mem::size_of::()) + + self.hash_set.capacity() * (std::mem::size_of::()) } /// remove the tmp dir @@ -157,11 +158,11 @@ impl _Cache for Caches { } } - fn get_hash(&self, offset: usize) -> Option { + fn get_hash(&self, offset: usize) -> Option { self.map_offset.get(&offset).map(|x| *x) } - fn insert(&self, offset: usize, hash: SHA1, obj: CacheObject) -> Arc { + fn insert(&self, offset: usize, hash: ObjectHash, obj: CacheObject) -> Arc { let obj_arc = Arc::new(obj); { // ? whether insert to cache directly or only write to tmp file @@ -190,7 +191,7 @@ impl _Cache for Caches { } } - fn get_by_hash(&self, hash: SHA1) -> Option> { + fn get_by_hash(&self, hash: ObjectHash) -> Option> { // check if the hash is in the cache( lru or tmp file) if self.hash_set.contains(&hash) { match self.try_get(hash) { @@ -236,12 +237,15 @@ mod test { use super::*; use crate::{ - hash::SHA1, + hash::{HashKind, ObjectHash, set_hash_kind_for_test}, internal::{object::types::ObjectType, pack::cache_object::CacheObjectInfo}, }; + use std::sync::Arc; + use std::thread; #[test] fn test_cache_single_thread() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); let tmp_path = source.clone().join("tests/.cache_tmp"); @@ -250,8 +254,8 @@ mod test { } let cache = Caches::new(Some(2048), tmp_path, 1); - let a_hash = SHA1::new(String::from("a").as_bytes()); - let b_hash = SHA1::new(String::from("b").as_bytes()); + let a_hash = ObjectHash::new(String::from("a").as_bytes()); + let b_hash = ObjectHash::new(String::from("b").as_bytes()); let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, a_hash), data_decompressed: vec![0; 800], @@ -277,7 +281,7 @@ mod test { assert!(cache.try_get(b_hash).is_some()); assert!(cache.try_get(a_hash).is_some()); - let c_hash = SHA1::new(String::from("c").as_bytes()); + let c_hash = ObjectHash::new(String::from("c").as_bytes()); // insert c which will evict both a and b let c = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, c_hash), @@ -292,4 +296,102 @@ mod test { assert!(cache.try_get(c_hash).is_some()); assert!(cache.get_by_hash(c_hash).is_some()); } + #[test] + fn test_cache_single_thread_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + let tmp_path = source.clone().join("tests/.cache_tmp_sha256"); + + if tmp_path.exists() { + fs::remove_dir_all(&tmp_path).unwrap(); + } + let cache = Caches::new(Some(4096), tmp_path, 1); + let a_hash = ObjectHash::new(String::from("a").as_bytes()); + let b_hash = ObjectHash::new(String::from("b").as_bytes()); + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, a_hash), + data_decompressed: vec![0; 1500], + mem_recorder: None, + offset: 0, + is_delta_in_pack: false, + }; + let b = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, b_hash), + data_decompressed: vec![0; 1500], + mem_recorder: None, + offset: 0, + is_delta_in_pack: false, + }; + // insert a + cache.insert(a.offset, a_hash, a.clone()); + assert!(cache.hash_set.contains(&a_hash)); + assert!(cache.try_get(a_hash).is_some()); + // insert b, a should still be in cache + cache.insert(b.offset, b_hash, b.clone()); + assert!(cache.hash_set.contains(&b_hash)); + assert!(cache.try_get(b_hash).is_some()); + assert!(cache.try_get(a_hash).is_some()); + let c_hash = ObjectHash::new(String::from("c").as_bytes()); + // insert c which will evict both a and b + let c = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, c_hash), + data_decompressed: vec![0; 3000], + mem_recorder: None, + offset: 0, + is_delta_in_pack: false, + }; + cache.insert(c.offset, c_hash, c.clone()); + assert!(cache.try_get(a_hash).is_none()); + assert!(cache.try_get(b_hash).is_none()); + assert!(cache.try_get(c_hash).is_some()); + assert!(cache.get_by_hash(c_hash).is_some()); + } + #[test] + /// consider the multi-threaded scenario where different threads use different hash kinds + fn test_cache_multi_thread_mixed_hash_kinds() { + let base = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + let tmp_path = base.join("tests/.cache_tmp_mixed"); + if tmp_path.exists() { + fs::remove_dir_all(&tmp_path).unwrap(); + } + + let cache = Arc::new(Caches::new(Some(4096), tmp_path, 2)); + + let cache_sha1 = Arc::clone(&cache); + let handle_sha1 = thread::spawn(move || { + let _g = set_hash_kind_for_test(HashKind::Sha1); + let hash = ObjectHash::new(b"sha1-entry"); + let obj = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash), + data_decompressed: vec![0; 800], + mem_recorder: None, + offset: 1, + is_delta_in_pack: false, + }; + cache_sha1.insert(obj.offset, hash, obj.clone()); + assert!(cache_sha1.hash_set.contains(&hash)); + assert!(cache_sha1.try_get(hash).is_some()); + }); + + let cache_sha256 = Arc::clone(&cache); + let handle_sha256 = thread::spawn(move || { + let _g = set_hash_kind_for_test(HashKind::Sha256); + let hash = ObjectHash::new(b"sha256-entry"); + let obj = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash), + data_decompressed: vec![0; 1500], + mem_recorder: None, + offset: 2, + is_delta_in_pack: false, + }; + cache_sha256.insert(obj.offset, hash, obj.clone()); + assert!(cache_sha256.hash_set.contains(&hash)); + assert!(cache_sha256.try_get(hash).is_some()); + }); + + handle_sha1.join().unwrap(); + handle_sha256.join().unwrap(); + + assert_eq!(cache.total_inserted(), 2); + } } diff --git a/src/internal/pack/cache_object.rs b/src/internal/pack/cache_object.rs index 146a3e80..b65975e2 100644 --- a/src/internal/pack/cache_object.rs +++ b/src/internal/pack/cache_object.rs @@ -12,7 +12,7 @@ use threadpool::ThreadPool; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::pack::entry::Entry; use crate::internal::pack::utils; -use crate::{hash::SHA1, internal::object::types::ObjectType}; +use crate::{hash::ObjectHash, internal::object::types::ObjectType}; // /// record heap-size of all CacheObjects, used for memory limit. // static CACHE_OBJS_MEM_SIZE: AtomicUsize = AtomicUsize::new(0); @@ -56,16 +56,16 @@ impl Deserialize<'a>> FileLoadStore for T { pub(crate) enum CacheObjectInfo { /// The object is one of the four basic types: /// [`ObjectType::Blob`], [`ObjectType::Tree`], [`ObjectType::Commit`], or [`ObjectType::Tag`]. - /// The metadata contains the [`ObjectType`] and the [`SHA1`] hash of the object. - BaseObject(ObjectType, SHA1), + /// The metadata contains the [`ObjectType`] and the [`ObjectHash`] hash of the object. + BaseObject(ObjectType, ObjectHash), /// The object is an offset delta with a specified offset delta [`usize`], /// and the size of the expanded object (previously `delta_final_size`). OffsetDelta(usize, usize), /// Similar to [`OffsetDelta`], but delta algorithm is `zstd`. OffsetZstdelta(usize, usize), - /// The object is a hash delta with a specified [`SHA1`] hash, + /// The object is a hash delta with a specified [`ObjectHash`] hash, /// and the size of the expanded object (previously `delta_final_size`). - HashDelta(SHA1, usize), + HashDelta(ObjectHash, usize), } impl CacheObjectInfo { @@ -197,10 +197,10 @@ impl CacheObject { self.info.object_type() } - /// Get the [`SHA1`] hash of the object. + /// Get the [`ObjectHash`] hash of the object. /// /// If the object is a delta object, return [`None`]. - pub fn base_object_hash(&self) -> Option { + pub fn base_object_hash(&self) -> Option { match &self.info { CacheObjectInfo::BaseObject(_, hash) => Some(*hash), _ => None, @@ -220,7 +220,7 @@ impl CacheObject { /// Get the hash delta of the object. /// /// If the object is not a hash delta, return [`None`]. - pub fn hash_delta(&self) -> Option { + pub fn hash_delta(&self) -> Option { match &self.info { CacheObjectInfo::HashDelta(hash, _) => Some(*hash), _ => None, @@ -370,11 +370,12 @@ mod test { use lru_mem::LruCache; use super::*; - + use crate::hash::{HashKind, set_hash_kind_for_test}; #[test] fn test_heap_size_record() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut obj = CacheObject { - info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, data_decompressed: vec![0; 1024], mem_recorder: None, @@ -388,11 +389,30 @@ mod test { drop(obj); assert_eq!(mem.load(Ordering::Relaxed), 0); } + #[test] + fn test_heap_size_record_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut obj = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + let mem = Arc::new(AtomicUsize::default()); + assert_eq!(mem.load(Ordering::Relaxed), 0); + obj.set_mem_recorder(mem.clone()); + obj.record_mem_size(); + assert_eq!(mem.load(Ordering::Relaxed), obj.mem_size()); + drop(obj); + assert_eq!(mem.load(Ordering::Relaxed), 0); + } #[test] fn test_cache_object_with_same_size() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let a = CacheObject { - info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, data_decompressed: vec![0; 1024], mem_recorder: None, @@ -404,13 +424,30 @@ mod test { let b = ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(false)), None); assert!(b.heap_size() == 1024); } + #[test] + fn test_cache_object_with_same_size_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + assert!(a.heap_size() == 2048); + + // let b = ArcWrapper(Arc::new(a.clone())); + let b = ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(false)), None); + assert!(b.heap_size() == 2048); + } #[test] fn test_cache_object_with_lru() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut cache = LruCache::new(2048); - let hash_a = SHA1::default(); - let hash_b = SHA1::new(b"b"); // whatever different hash + let hash_a = ObjectHash::default(); + let hash_b = ObjectHash::new(b"b"); // whatever different hash let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a), offset: 0, @@ -458,6 +495,60 @@ mod test { assert!(r.is_none()); } } + #[test] + fn test_cache_object_with_lru_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut cache = LruCache::new(4096); + + let hash_a = ObjectHash::default(); + let hash_b = ObjectHash::new(b"b"); // whatever different hash + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + println!("a.heap_size() = {}", a.heap_size()); + + let b = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_b), + offset: 0, + data_decompressed: vec![0; 3072], + mem_recorder: None, + is_delta_in_pack: false, + }; + { + let r = cache.insert( + hash_a.to_string(), + ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_ok()) + } + { + let r = cache.try_insert( + hash_b.to_string(), + ArcWrapper::new(Arc::new(b.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_err()); + if let Err(lru_mem::TryInsertError::WouldEjectLru { .. }) = r { + // 匹配到指定错误,不需要额外操作 + } else { + panic!("Expected WouldEjectLru error"); + } + // 使用不同的键插入b,这样a会被驱逐 + let r = cache.insert( + hash_b.to_string(), + ArcWrapper::new(Arc::new(b.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_ok()); + } + { + // a should be ejected + let r = cache.get(&hash_a.to_string()); + assert!(r.is_none()); + } + } #[derive(Serialize, Deserialize)] struct Test { @@ -523,8 +614,9 @@ mod test { #[test] fn test_cache_object_serialize() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let a = CacheObject { - info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, data_decompressed: vec![0; 1024], mem_recorder: None, @@ -538,6 +630,24 @@ mod test { assert_eq!(a.data_decompressed, b.data_decompressed); assert_eq!(a.offset, b.offset); } + #[test] + fn test_cache_object_serialize_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), + offset: 0, + data_decompressed: vec![0; 2048], + mem_recorder: None, + is_delta_in_pack: false, + }; + let s = bincode::serde::encode_to_vec(&a, bincode::config::standard()).unwrap(); + let b: CacheObject = bincode::serde::decode_from_slice(&s, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(a.info, b.info); + assert_eq!(a.data_decompressed, b.data_decompressed); + assert_eq!(a.offset, b.offset); + } #[test] fn test_arc_wrapper_drop_store() { diff --git a/src/internal/pack/decode.rs b/src/internal/pack/decode.rs index 6c1f920f..5864c4e9 100644 --- a/src/internal/pack/decode.rs +++ b/src/internal/pack/decode.rs @@ -14,7 +14,7 @@ use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; use crate::errors::GitError; -use crate::hash::{SHA1, get_hash_kind}; +use crate::hash::{ObjectHash, get_hash_kind, set_hash_kind}; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::zstdelta; @@ -79,7 +79,7 @@ impl Pack { let cache_mem_size = mem_limit.map(|mem_limit| mem_limit * 4 / 5); Pack { number: 0, - signature: SHA1::default(), + signature: ObjectHash::default(), objects: Vec::new(), pool: Arc::new(ThreadPool::new(thread_num)), waitlist: Arc::new(Waitlist::new()), @@ -309,9 +309,9 @@ impl Pack { }) } ObjectType::HashDelta => { - // Read 20 bytes to get the reference object SHA1 hash - let ref_sha1 = SHA1::from_stream(pack).unwrap(); - // Offset is incremented by 20 bytes + // Read 20/32 bytes to get the reference object SHA1/SHA256 hash + let ref_sha1 = ObjectHash::from_stream(pack).unwrap(); + // Offset is incremented by 20/32 bytes *offset += get_hash_kind().size(); let (data, raw_size) = Pack::decompress_data(pack, size)?; @@ -345,7 +345,7 @@ impl Pack { ) -> Result<(), GitError> where F: Fn(MetaAttached) + Sync + Send + 'static, - C: FnOnce(SHA1) + Send + 'static, + C: FnOnce(ObjectHash) + Send + 'static, { let time = Instant::now(); let mut last_update_time = time.elapsed().as_millis(); @@ -414,7 +414,9 @@ impl Pack { let caches = caches.clone(); let waitlist = self.waitlist.clone(); + let kind = get_hash_kind(); self.pool.execute(move || { + set_hash_kind(kind); match obj.info { CacheObjectInfo::BaseObject(_, _) => { Self::cache_obj_and_process_waitlist(params, obj); @@ -454,7 +456,7 @@ impl Pack { } log_info(i, self); let render_hash = reader.final_hash(); - self.signature = SHA1::from_stream(&mut reader).unwrap(); + self.signature = ObjectHash::from_stream(&mut reader).unwrap(); if render_hash != self.signature { return Err(GitError::InvalidPackFile(format!( @@ -503,7 +505,9 @@ impl Pack { mut pack: impl BufRead + Send + 'static, sender: UnboundedSender, ) -> JoinHandle { + let kind = get_hash_kind(); thread::spawn(move || { + set_hash_kind(kind); self.decode( &mut pack, move |entry| { @@ -511,7 +515,7 @@ impl Pack { eprintln!("Channel full, failed to send entry: {e:?}"); } }, - None::, + None::, ) .unwrap(); self @@ -523,8 +527,9 @@ impl Pack { mut self, mut stream: impl Stream> + Unpin + Send + 'static, sender: UnboundedSender>, - pack_hash_send: Option>, + pack_hash_send: Option>, ) -> Self { + let kind = get_hash_kind(); let (tx, rx) = std::sync::mpsc::channel(); let mut reader = StreamBufReader::new(rx); tokio::spawn(async move { @@ -539,6 +544,7 @@ impl Pack { // CPU-bound task, so use spawn_blocking // DO NOT use thread::spawn, because it will block tokio runtime (if single-threaded runtime, like in tests) tokio::task::spawn_blocking(move || { + set_hash_kind(kind); self.decode( &mut reader, move |entry: MetaAttached| { @@ -547,7 +553,7 @@ impl Pack { eprintln!("unbound channel Sending Error: {e:?}"); } }, - Some(move |pack_id: SHA1| { + Some(move |pack_id: ObjectHash| { if let Some(pack_id_send) = pack_hash_send && let Err(e) = pack_id_send.send(pack_id) { @@ -733,7 +739,7 @@ mod tests { use flate2::write::ZlibEncoder; use tokio_util::io::ReaderStream; - use crate::hash::SHA1; + use crate::hash::{HashKind, ObjectHash, set_hash_kind}; use crate::internal::pack::Pack; use crate::internal::pack::tests::init_logger; use futures_util::TryStreamExt; @@ -783,7 +789,22 @@ mod tests { let f = fs::File::open(source).unwrap(); let mut buffered = BufReader::new(f); let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + } + #[test] + fn test_pack_decode_without_delta_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); } #[test] @@ -799,11 +820,28 @@ mod tests { let f = fs::File::open(source).unwrap(); let mut buffered = BufReader::new(f); let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + } + #[test] + fn test_pack_decode_with_ref_delta_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + init_logger(); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); } #[test] fn test_pack_decode_no_mem_limit() { + let _guard = set_hash_kind(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/pack-1d0e6c14760c956c173ede71cb28f33d921e232f.pack"); @@ -812,11 +850,27 @@ mod tests { let f = fs::File::open(source).unwrap(); let mut buffered = BufReader::new(f); let mut p = Pack::new(None, None, Some(tmp), true); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + } + #[test] + fn test_pack_decode_no_mem_limit_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, None, Some(tmp), true); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); } #[tokio::test] async fn test_pack_decode_with_large_file_with_delta_without_ref() { + let _guard = set_hash_kind(HashKind::Sha1); init_logger(); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack"); @@ -836,7 +890,31 @@ mod tests { |_obj| { // println!("{:?} {}", obj.hash.to_string(), offset); }, - None::, + None::, + ); + if let Err(e) = rt { + fs::remove_dir_all(tmp).unwrap(); + panic!("Error: {e:?}"); + } + } // it will be stuck on dropping `Pack` on Windows if `mem_size` is None, so we need `mimalloc` + #[tokio::test] + async fn test_pack_decode_with_large_file_with_delta_without_ref_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + init_logger(); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(Some(20), Some(1024 * 1024), Some(tmp.clone()), true); + let rt = p.decode( + &mut buffered, + |_obj| { + // println!("{:?} {}", obj.hash.to_string(), offset); + }, + None::, ); if let Err(e) = rt { fs::remove_dir_all(tmp).unwrap(); @@ -878,6 +956,36 @@ mod tests { assert_eq!(count.load(Ordering::Acquire), p.number); assert_eq!(p.number, 358109); } + #[tokio::test] + async fn test_decode_large_file_stream_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + init_logger(); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + let f = tokio::fs::File::open(source).await.unwrap(); + let stream = ReaderStream::new(f).map_err(axum::Error::new); + let p = Pack::new(Some(20), Some(1024 * 1024), Some(tmp.clone()), true); + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let handle = tokio::spawn(async move { p.decode_stream(stream, tx, None).await }); + let count = Arc::new(AtomicUsize::new(0)); + let count_c = count.clone(); + // in tests, RUNTIME is single-threaded, so `sync code` will block the tokio runtime + let consume = tokio::spawn(async move { + let mut cnt = 0; + while let Some(_entry) = rx.recv().await { + cnt += 1; + } + tracing::info!("Received: {}", cnt); + count_c.store(cnt, Ordering::Release); + }); + let p = handle.await.unwrap(); + consume.await.unwrap(); + assert_eq!(count.load(Ordering::Acquire), p.number); + assert_eq!(p.number, 26); + } #[tokio::test] async fn test_decode_large_file_async() { @@ -903,9 +1011,30 @@ mod tests { let p = handle.join().unwrap(); assert_eq!(cnt, p.number); } + #[tokio::test] + async fn test_decode_large_file_async_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + let f = fs::File::open(source).unwrap(); + let buffered = BufReader::new(f); + let p = Pack::new(Some(20), Some(1024 * 1024), Some(tmp.clone()), true); + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let handle = p.decode_async(buffered, tx); // new thread + let mut cnt = 0; + while let Some(_entry) = rx.recv().await { + cnt += 1; //use entry here + } + let p = handle.join().unwrap(); + assert_eq!(cnt, p.number); + } #[test] fn test_pack_decode_with_delta_without_ref() { + let _guard = set_hash_kind(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/packs/pack-d50df695086eea6253a237cb5ac44af1629e7ced.pack"); @@ -915,12 +1044,30 @@ mod tests { let mut buffered = BufReader::new(f); let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); print!("pack_id: {:?}", p.signature); - p.decode(&mut buffered, |_| {}, None::).unwrap(); + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); + print!("pack_id: {:?}", p.signature.to_string()); + } + #[test] + fn test_pack_decode_with_delta_without_ref_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + print!("pack_id: {:?}", p.signature); //default sha1 + p.decode(&mut buffered, |_| {}, None::) + .unwrap(); print!("pack_id: {:?}", p.signature.to_string()); } #[test] // Take too long time fn test_pack_decode_multi_task_with_large_file_with_delta_without_ref() { + let _guard = set_hash_kind(HashKind::Sha1); let task1 = std::thread::spawn(|| { test_pack_decode_with_large_file_with_delta_without_ref(); }); @@ -928,6 +1075,19 @@ mod tests { test_pack_decode_with_large_file_with_delta_without_ref(); }); + task1.join().unwrap(); + task2.join().unwrap(); + } + #[test] + fn test_pack_decode_multi_task_with_large_file_with_delta_without_ref_sha256() { + let _guard = set_hash_kind(HashKind::Sha256); + let task1 = std::thread::spawn(|| { + test_pack_decode_with_large_file_with_delta_without_ref_sha256(); + }); + let task2 = std::thread::spawn(|| { + test_pack_decode_with_large_file_with_delta_without_ref_sha256(); + }); + task1.join().unwrap(); task2.join().unwrap(); } diff --git a/src/internal/pack/encode.rs b/src/internal/pack/encode.rs index f676af90..a73d0edf 100644 --- a/src/internal/pack/encode.rs +++ b/src/internal/pack/encode.rs @@ -7,7 +7,11 @@ use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::object::types::ObjectType; use crate::time_it; use crate::zstdelta; -use crate::{errors::GitError, hash::SHA1, internal::pack::entry::Entry}; +use crate::{ + errors::GitError, + hash::{HashKind, ObjectHash, get_hash_kind}, + internal::pack::entry::Entry, +}; use ahash::AHasher; use flate2::write::ZlibEncoder; use natord::compare; @@ -22,6 +26,35 @@ const MAX_CHAIN_LEN: usize = 50; const MIN_DELTA_RATE: f64 = 0.5; // minimum delta rate //const MAX_ZSTDELTA_CHAIN_LEN: usize = 50; +/// Different hash algorithm enum +#[derive(Clone)] +enum Hashalgorithm { + Sha1(Sha1), + Sha256(sha2::Sha256), + // Future: support other hash algorithms +} +impl Hashalgorithm { + /// Update hash with data + fn update(&mut self, data: &[u8]) { + match self { + Hashalgorithm::Sha1(hasher) => hasher.update(data), + Hashalgorithm::Sha256(hasher) => hasher.update(data), + } + } + /// Finalize and get hash result + fn finalize(self) -> Vec { + match self { + Hashalgorithm::Sha1(hasher) => hasher.finalize().to_vec(), + Hashalgorithm::Sha256(hasher) => hasher.finalize().to_vec(), + } + } + fn new() -> Self { + match get_hash_kind() { + HashKind::Sha1 => Hashalgorithm::Sha1(Sha1::new()), + HashKind::Sha256 => Hashalgorithm::Sha256(sha2::Sha256::new()), + } + } +} /// A encoder for generating pack files with delta objects. pub struct PackEncoder { object_number: usize, @@ -29,9 +62,9 @@ pub struct PackEncoder { window_size: usize, // window: VecDeque<(Entry, usize)>, // entry and offset sender: Option>>, - inner_offset: usize, // offset of current entry - inner_hash: Sha1, // Not SHA1 because need update trait - final_hash: Option, + inner_offset: usize, // offset of current entry + inner_hash: Hashalgorithm, // introduce different hash algorithm + final_hash: Option, start_encoding: bool, } @@ -184,8 +217,8 @@ impl PackEncoder { process_index: 0, // window: VecDeque::with_capacity(window_size), sender: Some(sender), - inner_offset: 12, // 12 bytes header - inner_hash: Sha1::new(), + inner_offset: 12, // 12 bytes header + inner_hash: Hashalgorithm::new(), // introduce different hash algorithm final_hash: None, start_encoding: false, } @@ -202,7 +235,7 @@ impl PackEncoder { } /// Get the hash of the pack file. if the pack file is not finished, return None - pub fn get_hash(&self) -> Option { + pub fn get_hash(&self) -> Option { self.final_hash } @@ -341,7 +374,7 @@ impl PackEncoder { // Hash signature let hash_result = self.inner_hash.clone().finalize(); - self.final_hash = Some(SHA1::from_bytes(&hash_result).unwrap()); + self.final_hash = Some(ObjectHash::from_bytes(&hash_result).unwrap()); self.send_data(hash_result.to_vec()).await; self.drop_sender(); @@ -535,7 +568,7 @@ impl PackEncoder { // hash signature let hash_result = self.inner_hash.clone().finalize(); - self.final_hash = Some(SHA1::from_bytes(&hash_result).unwrap()); + self.final_hash = Some(ObjectHash::from_bytes(&hash_result).unwrap()); self.send_data(hash_result.to_vec()).await; self.drop_sender(); Ok(()) @@ -585,13 +618,13 @@ mod tests { use std::{io::Cursor, path::PathBuf}; use tokio::sync::Mutex; + use super::*; + use crate::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; use crate::internal::object::blob::Blob; use crate::internal::pack::utils::read_offset_encoding; use crate::internal::pack::{Pack, tests::init_logger}; use crate::time_it; - use super::*; - fn check_format(data: &Vec) { let mut p = Pack::new( None, @@ -601,12 +634,13 @@ mod tests { ); let mut reader = Cursor::new(data); tracing::debug!("start check format"); - p.decode(&mut reader, |_| {}, None::) + p.decode(&mut reader, |_| {}, None::) .expect("pack file format error"); } #[tokio::test] async fn test_pack_encoder() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); async fn encode_once(window_size: usize) -> Vec { let (tx, mut rx) = mpsc::channel(100); let (entry_tx, entry_rx) = mpsc::channel::>(1); @@ -646,6 +680,48 @@ mod tests { assert!(pack_with_delta.len() <= pack_without_delta_size); check_format(&pack_with_delta); } + #[tokio::test] + async fn test_pack_encoder_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + + async fn encode_once(window_size: usize) -> Vec { + let (tx, mut rx) = mpsc::channel(100); + let (entry_tx, entry_rx) = mpsc::channel::>(1); + + let str_vec = vec!["hello, word", "hello, world.", "!", "123141251251"]; + let encoder = PackEncoder::new(str_vec.len(), window_size, tx); + encoder.encode_async(entry_rx).await.unwrap(); + + for s in str_vec { + let blob = Blob::from_content(s); + let entry: Entry = blob.into(); + entry_tx + .send(MetaAttached { + inner: entry, + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + result + } + + // without delta + let pack_without_delta = encode_once(0).await; + let pack_without_delta_size = pack_without_delta.len(); + check_format(&pack_without_delta); + + // with delta + let pack_with_delta = encode_once(4).await; + assert!(pack_with_delta.len() <= pack_without_delta_size); + check_format(&pack_with_delta); + } async fn get_entries_for_test() -> Arc>> { let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -664,7 +740,33 @@ mod tests { let mut entries = entries_clone.blocking_lock(); entries.push(entry.inner); }, - None::, + None::, + ) + .unwrap(); + assert_eq!(p.number, entries.lock().await.len()); + tracing::info!("total entries: {}", p.number); + drop(p); + + entries + } + async fn get_entries_for_test_sha256() -> Arc>> { + let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/packs/pack-78047853c60a1a3bb587f59598bdeb773fefc821f6f60f4f4797644ad43dad3d.pack"); + + let mut p = Pack::new(None, None, Some(PathBuf::from("/tmp/.cache_temp")), true); + + let f = std::fs::File::open(&source).unwrap(); + tracing::info!("pack file size: {}", f.metadata().unwrap().len()); + let mut reader = std::io::BufReader::new(f); + let entries = Arc::new(Mutex::new(Vec::new())); + let entries_clone = entries.clone(); + p.decode( + &mut reader, + move |entry| { + let mut entries = entries_clone.blocking_lock(); + entries.push(entry.inner); + }, + None::, ) .unwrap(); assert_eq!(p.number, entries.lock().await.len()); @@ -676,6 +778,7 @@ mod tests { #[tokio::test] async fn test_pack_encoder_parallel_large_file() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let start = Instant::now(); @@ -735,9 +838,70 @@ mod tests { // check format check_format(&result); } + #[tokio::test] + async fn test_pack_encoder_parallel_large_file_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + + let start = Instant::now(); + // use sha256 pack file for testing + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let (tx, mut rx) = mpsc::channel(1_000_000); + let (entry_tx, entry_rx) = mpsc::channel::>(1_000_000); + + let mut encoder = PackEncoder::new(entries_number, 0, tx); + tokio::spawn(async move { + time_it!("test parallel encode sha256", { + encoder.parallel_encode(entry_rx).await.unwrap(); + }); + }); + + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("sha256 test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", result.len()); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + check_format(&result); + } #[tokio::test] async fn test_pack_encoder_large_file() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let entries = get_entries_for_test().await; let entries_number = entries.lock().await.len(); @@ -804,9 +968,79 @@ mod tests { total_original_size.saturating_sub(pack_size) ); } + #[tokio::test] + async fn test_pack_encoder_large_file_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let start = Instant::now(); + // encode entries + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::>(100_000); + + let mut encoder = PackEncoder::new(entries_number, 0, tx); + tokio::spawn(async move { + time_it!("test encode no parallel sha256", { + encoder.encode(entry_rx).await.unwrap(); + }); + }); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + // // only receive data + // while (rx.recv().await).is_some() { + // // do nothing + // } + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + } #[tokio::test] async fn test_pack_encoder_with_zstdelta() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let entries = get_entries_for_test().await; let entries_number = entries.lock().await.len(); @@ -866,6 +1100,68 @@ mod tests { // check format check_format(&result); } + #[tokio::test] + async fn test_pack_encoder_with_zstdelta_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let start = Instant::now(); + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::>(100_000); + + let encoder = PackEncoder::new(entries_number, 10, tx); + encoder.encode_async_with_zstdelta(entry_rx).await.unwrap(); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + + // check format + check_format(&result); + } #[test] fn test_encode_offset() { @@ -882,6 +1178,7 @@ mod tests { #[tokio::test] async fn test_pack_encoder_large_file_with_delta() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); init_logger(); let entries = get_entries_for_test().await; let entries_number = entries.lock().await.len(); @@ -939,6 +1236,69 @@ mod tests { total_original_size.saturating_sub(pack_size) ); + // check format + check_format(&result); + } + #[tokio::test] + async fn test_pack_encoder_large_file_with_delta_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + init_logger(); + let entries = get_entries_for_test_sha256().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::>(100_000); + + let encoder = PackEncoder::new(entries_number, 10, tx); + + let start = Instant::now(); // 开始时间 + encoder.encode_async(entry_rx).await.unwrap(); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx + .send(MetaAttached { + inner: entry.clone(), + meta: EntryMeta::new(), + }) + .await + .unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + // check format check_format(&result); } diff --git a/src/internal/pack/entry.rs b/src/internal/pack/entry.rs index f18269e1..627d1e01 100644 --- a/src/internal/pack/entry.rs +++ b/src/internal/pack/entry.rs @@ -2,7 +2,7 @@ use std::hash::{Hash, Hasher}; use serde::{Deserialize, Serialize}; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::object::blob::Blob; use crate::internal::object::commit::Commit; @@ -17,7 +17,7 @@ use crate::internal::object::types::ObjectType; pub struct Entry { pub obj_type: ObjectType, pub data: Vec, - pub hash: SHA1, + pub hash: ObjectHash, pub chain_len: usize, } diff --git a/src/internal/pack/mod.rs b/src/internal/pack/mod.rs index 3307f838..a4f71f9a 100644 --- a/src/internal/pack/mod.rs +++ b/src/internal/pack/mod.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicUsize; use threadpool::ThreadPool; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::internal::pack::cache::Caches; use crate::internal::pack::waitlist::Waitlist; @@ -24,7 +24,7 @@ use crate::internal::pack::waitlist::Waitlist; const DEFAULT_TMP_DIR: &str = "./.cache_temp"; pub struct Pack { pub number: usize, - pub signature: SHA1, + pub signature: ObjectHash, pub objects: Vec>, pub pool: Arc, pub waitlist: Arc, diff --git a/src/internal/pack/utils.rs b/src/internal/pack/utils.rs index bb32da3f..2a653146 100644 --- a/src/internal/pack/utils.rs +++ b/src/internal/pack/utils.rs @@ -3,7 +3,7 @@ use std::fs; use std::io::{self, Read}; use std::path::Path; -use crate::hash::SHA1; +use crate::hash::{ObjectHash, get_hash_kind}; use crate::internal::object::types::ObjectType; /// Checks if the reader has reached EOF (end of file). @@ -258,19 +258,36 @@ pub fn read_delta_object_size(stream: &mut R) -> io::Result<(usize, usi /// Calculate the SHA1 hash of the given object. ///
"` \0`" ///
data: The decompressed content of the object -pub fn calculate_object_hash(obj_type: ObjectType, data: &Vec) -> SHA1 { - let mut hash = Sha1::new(); - // Header: " \0" - hash.update(obj_type.to_bytes()); - hash.update(b" "); - hash.update(data.len().to_string()); - hash.update(b"\0"); - - // Decompressed data(raw content) - hash.update(data); - - let re: [u8; 20] = hash.finalize().into(); - SHA1::from_bytes(&re).unwrap() +pub fn calculate_object_hash(obj_type: ObjectType, data: &Vec) -> ObjectHash { + match get_hash_kind() { + crate::hash::HashKind::Sha1 => { + let mut hash = Sha1::new(); + // Header: " \0" + hash.update(obj_type.to_bytes()); + hash.update(b" "); + hash.update(data.len().to_string()); + hash.update(b"\0"); + + // Decompressed data(raw content) + hash.update(data); + + let re: [u8; 20] = hash.finalize().into(); + ObjectHash::Sha1(re) + } + crate::hash::HashKind::Sha256 => { + let mut hash = sha2::Sha256::new(); + // Header: " \0" + hash.update(obj_type.to_bytes()); + hash.update(b" "); + hash.update(data.len().to_string()); + hash.update(b"\0"); + + // Decompressed data(raw content) + hash.update(data); + let re: [u8; 32] = hash.finalize().into(); + ObjectHash::Sha256(re) + } + } } /// Create an empty directory or clear the existing directory. pub fn create_empty_dir>(path: P) -> io::Result<()> { @@ -314,6 +331,7 @@ macro_rules! time_it { #[cfg(test)] mod tests { + use crate::hash::{HashKind, set_hash_kind_for_test}; use crate::internal::object::types::ObjectType; use std::io; use std::io::Cursor; @@ -323,9 +341,19 @@ mod tests { #[test] fn test_calc_obj_hash() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let hash = calculate_object_hash(ObjectType::Blob, &b"a".to_vec()); assert_eq!(hash.to_string(), "2e65efe2a145dda7ee51d1741299f848e5bf752e"); } + #[test] + fn test_calc_obj_hash_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let hash = calculate_object_hash(ObjectType::Blob, &b"a".to_vec()); + assert_eq!( + hash.to_string(), + "eb337bcee2061c5313c9a1392116b6c76039e9e30d71467ae359b36277e17dc7" + ); + } #[test] fn eof() { diff --git a/src/internal/pack/waitlist.rs b/src/internal/pack/waitlist.rs index 13107b79..a678ac6f 100644 --- a/src/internal/pack/waitlist.rs +++ b/src/internal/pack/waitlist.rs @@ -1,4 +1,4 @@ -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::pack::cache_object::CacheObject; use dashmap::DashMap; @@ -8,7 +8,7 @@ use dashmap::DashMap; pub struct Waitlist { //TODO Memory Control! pub map_offset: DashMap>, - pub map_ref: DashMap>, + pub map_ref: DashMap>, } impl Waitlist { @@ -20,13 +20,13 @@ impl Waitlist { self.map_offset.entry(offset).or_default().push(obj); } - pub fn insert_ref(&self, hash: SHA1, obj: CacheObject) { + pub fn insert_ref(&self, hash: ObjectHash, obj: CacheObject) { self.map_ref.entry(hash).or_default().push(obj); } /// Take objects out (get & remove) ///
Return Vec::new() if None - pub fn take(&self, offset: usize, hash: SHA1) -> Vec { + pub fn take(&self, offset: usize, hash: ObjectHash) -> Vec { let mut res = Vec::new(); if let Some((_, vec)) = self.map_offset.remove(&offset) { res.extend(vec); diff --git a/src/internal/pack/wrapper.rs b/src/internal/pack/wrapper.rs index 818d63ab..ba40f315 100644 --- a/src/internal/pack/wrapper.rs +++ b/src/internal/pack/wrapper.rs @@ -2,19 +2,26 @@ use std::io::{self, BufRead, Read}; use sha1::{Digest, Sha1}; -use crate::hash::SHA1; +use crate::hash::{HashKind, ObjectHash, get_hash_kind}; -/// [`Wrapper`] is a wrapper around a reader that also computes the SHA1 hash of the data read. +/// [`Wrapper`] is a wrapper around a reader that also computes the SHA1/ SHA256 hash of the data read. /// /// It is designed to work with any reader that implements `BufRead`. /// /// Fields: /// * `inner`: The inner reader. -/// * `hash`: The SHA1 hash state. +/// * `hash`: The hash state. /// * `count_hash`: A flag to indicate whether to compute the hash while reading. +/// +/// we abstract the hasher to support both SHA1 and SHA256 Now. +#[derive(Clone)] +enum Hasher { + Sha1(Sha1), + Sha256(sha2::Sha256), +} pub struct Wrapper { inner: R, - hash: Sha1, + hash: Hasher, bytes_read: usize, } @@ -30,7 +37,10 @@ where pub fn new(inner: R) -> Self { Self { inner, - hash: Sha1::new(), // Initialize a new SHA1 hasher + hash: match get_hash_kind() { + HashKind::Sha1 => Hasher::Sha1(Sha1::new()), + HashKind::Sha256 => Hasher::Sha256(sha2::Sha256::new()), + }, // Initialize a new SHA1/ SHA256 hasher bytes_read: 0, } } @@ -39,12 +49,20 @@ where self.bytes_read } - /// Returns the final SHA1 hash of the data read so far. + /// Returns the final SHA1/ SHA256 hash of the data read so far. /// - /// This is a clone of the internal hash state finalized into a SHA1 hash. - pub fn final_hash(&self) -> SHA1 { - let re: [u8; 20] = self.hash.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes - SHA1::from_bytes(&re).unwrap() + /// This is a clone of the internal hash state finalized into a SHA1/ SHA256 hash. + pub fn final_hash(&self) -> ObjectHash { + match &self.hash.clone() { + Hasher::Sha1(hasher) => { + let re: [u8; 20] = hasher.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes + ObjectHash::from_bytes(&re).unwrap() + } + Hasher::Sha256(hasher) => { + let re: [u8; 32] = hasher.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes + ObjectHash::from_bytes(&re).unwrap() + } + } } } @@ -63,7 +81,10 @@ where /// * `amt`: The amount of data to consume from the buffer. fn consume(&mut self, amt: usize) { let buffer = self.inner.fill_buf().expect("Failed to fill buffer"); - self.hash.update(&buffer[..amt]); // Update hash with the data being consumed + match &mut self.hash { + Hasher::Sha1(hasher) => hasher.update(&buffer[..amt]), // Update SHA1 hash with the data being consumed + Hasher::Sha256(hasher) => hasher.update(&buffer[..amt]), // Update SHA256 hash with the data being consumed + } self.inner.consume(amt); // Consume the data from the inner reader self.bytes_read += amt; } @@ -83,7 +104,10 @@ where /// Returns the number of bytes read. fn read(&mut self, buf: &mut [u8]) -> io::Result { let o = self.inner.read(buf)?; // Read data into the buffer - self.hash.update(&buf[..o]); // Update hash with the data being read + match &mut self.hash { + Hasher::Sha1(hasher) => hasher.update(&buf[..o]), // Update SHA1 hash with the data being read + Hasher::Sha256(hasher) => hasher.update(&buf[..o]), // Update SHA256 hash with the data being read + } self.bytes_read += o; Ok(o) // Return the number of bytes read } @@ -95,10 +119,25 @@ mod tests { use sha1::{Digest, Sha1}; + use crate::hash::{HashKind, set_hash_kind}; use crate::internal::pack::wrapper::Wrapper; - #[test] fn test_wrapper_read() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha1); + let data = b"Hello, world!"; // Sample data + let cursor = Cursor::new(data.as_ref()); + let buf_reader = BufReader::new(cursor); + let mut wrapper = Wrapper::new(buf_reader); + + let mut buffer = vec![0; data.len()]; + wrapper.read_exact(&mut buffer)?; + + assert_eq!(buffer, data); + Ok(()) + } + #[test] + fn test_wrapper_read_sha256() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha256); let data = b"Hello, world!"; // Sample data let cursor = Cursor::new(data.as_ref()); let buf_reader = BufReader::new(cursor); @@ -113,6 +152,7 @@ mod tests { #[test] fn test_wrapper_hash() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha1); let data = b"Hello, world!"; let cursor = Cursor::new(data.as_ref()); let buf_reader = BufReader::new(cursor); @@ -129,4 +169,23 @@ mod tests { assert_eq!(hash_result.as_ref(), expected_hash); Ok(()) } + #[test] + fn test_wrapper_hash_sha256() -> io::Result<()> { + let _guard = set_hash_kind(HashKind::Sha256); + let data = b"Hello, world!"; + let cursor = Cursor::new(data.as_ref()); + let buf_reader = BufReader::new(cursor); + let mut wrapper = Wrapper::new(buf_reader); + + let mut buffer = vec![0; data.len()]; + wrapper.read_exact(&mut buffer)?; + + let hash_result = wrapper.final_hash(); + let mut hasher = sha2::Sha256::new(); + hasher.update(data); + let expected_hash: [u8; 32] = hasher.finalize().into(); + + assert_eq!(hash_result.as_ref(), &expected_hash); + Ok(()) + } } diff --git a/tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack b/tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack new file mode 100644 index 00000000..e8f97153 --- /dev/null +++ b/tests/data/packs/git-large-sha256-f6455f09d816f54d115724975da6e7edfb100d746ad145bfd0d2ddc0e0261f5d.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bdad72b0d7ee8a375526ef57efb10963924fa484a8900e00a66481d8f40eb19 +size 20293 diff --git a/tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack b/tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack new file mode 100644 index 00000000..6be61a44 --- /dev/null +++ b/tests/data/packs/pack-delta-sha256-3662654057d1adedf50f2a80bfdec752e00ac72798859d850ab1dcfd801beedd.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df0a36c6ca8c7d4324848c4502f1918e44a04a359b6f8ed255b296c91ed89821 +size 23190 diff --git a/tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack b/tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack new file mode 100644 index 00000000..c2d3f9da --- /dev/null +++ b/tests/data/packs/ref-delta-0e26651d43b149c9baef6035c19cca140f82bb0d0cc5b12fda0ae89ff6a25195.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe0cea185b8622455c0d66089e693b4fabc662ac3d82dd7b10d5815fd43ba31 +size 4904 From 7f2807a11225786c9af75109bf38ab50a8f668f7 Mon Sep 17 00:00:00 2001 From: jackieismpc Date: Wed, 26 Nov 2025 20:04:58 +0800 Subject: [PATCH 4/7] 1.Completed updates to the index module, but tests are still failing due to existing parsing issues in Index::from_file 2.Added SHA-256 index test files 3.Plan to perform a small-scale refactor of the utility functions next Signed-off-by: jackieismpc --- src/internal/index.rs | 81 ++++++++++++++++++++++++++++------- src/utils.rs | 6 +-- tests/data/index/index-9-256 | Bin 0 -> 826 bytes 3 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 tests/data/index/index-9-256 diff --git a/src/internal/index.rs b/src/internal/index.rs index 5c94acf4..40b7e170 100644 --- a/src/internal/index.rs +++ b/src/internal/index.rs @@ -12,10 +12,47 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use sha1::{Digest, Sha1}; use crate::errors::GitError; -use crate::hash::{SHA1, get_hash_kind}; +use crate::hash::{ObjectHash, get_hash_kind, HashKind}; use crate::internal::pack::wrapper::Wrapper; use crate::utils; +/// Different hash algorithm enum +#[derive(Clone)] +enum Hashalgorithm { + Sha1(Sha1), + Sha256(sha2::Sha256), + // Future: support other hash algorithms +} +impl Hashalgorithm { + /// Update hash with data + fn update(&mut self, data: &[u8]) { + match self { + Hashalgorithm::Sha1(hasher) => hasher.update(data), + Hashalgorithm::Sha256(hasher) => hasher.update(data), + } + } + /// Finalize and get hash result + fn finalize(self) -> Vec { + match self { + Hashalgorithm::Sha1(hasher) => hasher.finalize().to_vec(), + Hashalgorithm::Sha256(hasher) => hasher.finalize().to_vec(), + } + } + fn new() -> Self { + match get_hash_kind() { + HashKind::Sha1 => Hashalgorithm::Sha1(Sha1::new()), + HashKind::Sha256 => Hashalgorithm::Sha256(sha2::Sha256::new()), + } + } +} +impl std::io::Write for Hashalgorithm { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.update(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } +} + #[derive(PartialEq, Eq, Debug, Clone)] pub struct Time { seconds: u32, @@ -112,7 +149,7 @@ pub struct IndexEntry { pub uid: u32, // 0 for windows pub gid: u32, // 0 for windows pub size: u32, - pub hash: SHA1, + pub hash: ObjectHash, pub flags: Flags, pub name: String, } @@ -138,7 +175,7 @@ impl Display for IndexEntry { impl IndexEntry { /** Metadata must be got by [fs::symlink_metadata] to avoid following symlink */ - pub fn new(meta: &fs::Metadata, hash: SHA1, name: String) -> Self { + pub fn new(meta: &fs::Metadata, hash: ObjectHash, name: String) -> Self { let mut entry = IndexEntry { ctime: Time::from_system_time(meta.created().unwrap()), mtime: Time::from_system_time(meta.modified().unwrap()), @@ -181,7 +218,7 @@ impl IndexEntry { /// - `file`: **to workdir path** /// - `workdir`: absolute or relative path - pub fn new_from_file(file: &Path, hash: SHA1, workdir: &Path) -> io::Result { + pub fn new_from_file(file: &Path, hash: ObjectHash, workdir: &Path) -> io::Result { let name = file.to_str().unwrap().to_string(); let file_abs = workdir.join(file); let meta = fs::symlink_metadata(file_abs)?; // without following symlink @@ -189,7 +226,7 @@ impl IndexEntry { Ok(index) } - pub fn new_from_blob(name: String, hash: SHA1, size: u32) -> Self { + pub fn new_from_blob(name: String, hash: ObjectHash, size: u32) -> Self { IndexEntry { ctime: Time { seconds: 0, @@ -266,7 +303,7 @@ impl Index { uid: file.read_u32::()?, gid: file.read_u32::()?, size: file.read_u32::()?, - hash: utils::read_sha1(file)?, + hash: utils::read_sha(file)?, flags: Flags::from(file.read_u16::()?), name: String::new(), }; @@ -282,7 +319,8 @@ impl Index { // 1-8 nul bytes as necessary to pad the entry to a multiple of eight bytes // while keeping the name NUL-terminated. // so at least 1 byte nul - let padding = 8 - ((22 + name_len) % 8); // 22 = sha1 + flags, others are 40 % 8 == 0 + let hash_len = get_hash_kind().size(); + let padding = (8 - ((hash_len + 2 + name_len) % 8)) % 8; // 2 for flags utils::read_bytes(file, padding)?; } @@ -310,7 +348,7 @@ impl Index { // check sum let file_hash = file.final_hash(); - let check_sum = utils::read_sha1(file)?; + let check_sum = utils::read_sha(file)?; if file_hash != check_sum { return Err(GitError::InvalidIndexFile("Check sum failed".to_string())); } @@ -381,11 +419,11 @@ impl Index { )); let new_size = meta.len() as u32; - // re-calculate SHA1 + // re-calculate SHA1/SHA256 let mut file = File::open(&abs_path)?; - let mut hasher = Sha1::new(); + let mut hasher = Hashalgorithm::new(); io::copy(&mut file, &mut hasher)?; - let new_hash = SHA1::from_bytes(&hasher.finalize()).unwrap(); + let new_hash = ObjectHash::from_bytes(&hasher.finalize()).unwrap(); // refresh index if entry.ctime != new_ctime @@ -456,11 +494,11 @@ impl Index { self.entries.contains_key(&(name.to_string(), stage)) } - pub fn get_hash(&self, file: &str, stage: u8) -> Option { + pub fn get_hash(&self, file: &str, stage: u8) -> Option { self.get(file, stage).map(|entry| entry.hash) } - pub fn verify_hash(&self, file: &str, stage: u8, hash: &SHA1) -> bool { + pub fn verify_hash(&self, file: &str, stage: u8, hash: &ObjectHash) -> bool { let inner_hash = self.get_hash(file, stage); if let Some(inner_hash) = inner_hash { &inner_hash == hash @@ -541,7 +579,7 @@ impl Index { #[cfg(test)] mod tests { use super::*; - + use crate::hash::{set_hash_kind_for_test,HashKind}; #[test] fn test_time() { let time = Time { @@ -565,6 +603,7 @@ mod tests { #[test] fn test_index() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/index/index-760"); @@ -574,6 +613,18 @@ mod tests { println!("{entry}"); } } + #[test] + fn test_index_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("tests/data/index/index-9-256"); + + let index = Index::from_file(source).unwrap();//index-里有文件名不是 UTF‑8,Index::from_file 里用 String::from_utf8(name)? 直接就报错了 + assert_eq!(index.size(), 9); + for (_, entry) in index.entries.iter() { + println!("{entry}"); + } + } #[test] fn test_index_to_file() { @@ -592,7 +643,7 @@ mod tests { source.push("Cargo.toml"); let file = Path::new(source.as_path()); // use as a normal file - let hash = SHA1::from_bytes(&[0; 20]).unwrap(); + let hash = ObjectHash::from_bytes(&[0; 20]).unwrap(); let workdir = Path::new("../"); let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); println!("{entry}"); diff --git a/src/utils.rs b/src/utils.rs index 2ed62f41..b6afcfc8 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,4 @@ -use crate::hash::SHA1; +use crate::hash::ObjectHash; use std::io; use std::io::{BufRead, Read}; @@ -8,8 +8,8 @@ pub fn read_bytes(file: &mut impl Read, len: usize) -> io::Result> { Ok(buf) } -pub fn read_sha1(file: &mut impl Read) -> io::Result { - SHA1::from_stream(file) +pub fn read_sha(file: &mut impl Read) -> io::Result { + ObjectHash::from_stream(file) } /// A lightweight wrapper that counts bytes read from the underlying reader. diff --git a/tests/data/index/index-9-256 b/tests/data/index/index-9-256 new file mode 100644 index 0000000000000000000000000000000000000000..3e6b8b7d7765ad2b560d10d2855ecb72e7f53e99 GIT binary patch literal 826 zcmZ?q402{*U|<4b&P=rz@+L`FSz$CI0|PI^?mhhs42??|7#P0-)rbHwo6e6Lw(}lM zb>8gNUR0v^_s#alpBFY?Hcy_QdM`Y8=j$yDtciLh6(tN{b5g(0fYDI%CZL=5fG6N# zmhR{N-s~To4;$AqdEClNm;4+*zb^IKH#W)n^$e^@Q1fOBaXZOYWU4`Euz3e#-=dgz zfPrD#!CgTT?vvKOEO*r2Bcj^=^%5K3j-B-%h1dlwF8=7{VPH?nOxH`w%maD`1Ts}a zJ@^~DaWnkb@475G%n?G-p`)zBVAkpTjH~kyt>{^t}eQEiUlAD|7uV!FP zhPrdkPBr6#j7&8M4RL4s3RLq7J)T)vSHAw()w8&Gb=YuP&;A!xKn@&3*N@0M%Ru7WZpwR;Y*l*GqhAs<8jQ{HmO` zo5uS*LSCD+@PuE~<7ow&n^~j}aVx~!f-M$nr~b@TgV11e51e8}Ggr*-!xPmMub)r9 z`|c=GoAB<0xLf*zsivVd&6Z1IN;~IeF>s`1=A?py0AwsMT$i&66?2}1(O`4;F1vze zF5&dVospW8ldoHzUzC#qHn(V}7`MvAOjQUCHg`|YCp2?KX0KcqrqLF(-{?chPo>AT zc2n Date: Wed, 26 Nov 2025 20:45:39 +0800 Subject: [PATCH 5/7] 1.Completed refactor of the utility functions 2.Finished all changes outside the protocol module Signed-off-by: jackieismpc --- src/diff.rs | 69 ++++++++++++++++++++++--------------- src/internal/index.rs | 58 +++++++++---------------------- src/internal/pack/encode.rs | 35 ++----------------- src/utils.rs | 42 ++++++++++++++++++++-- 4 files changed, 100 insertions(+), 104 deletions(-) diff --git a/src/diff.rs b/src/diff.rs index 5b542ad7..18189c4e 100644 --- a/src/diff.rs +++ b/src/diff.rs @@ -1,4 +1,4 @@ -use crate::hash::SHA1; +use crate::hash::ObjectHash; use path_absolutize::Absolutize; use similar::{Algorithm, ChangeTag, TextDiff}; use std::collections::{HashMap, HashSet, VecDeque}; @@ -87,13 +87,13 @@ impl Diff { /// Compute diffs for a set of files, honoring an optional filter and emitting unified diffs. pub fn diff( - old_blobs: Vec<(PathBuf, SHA1)>, - new_blobs: Vec<(PathBuf, SHA1)>, + old_blobs: Vec<(PathBuf, ObjectHash)>, + new_blobs: Vec<(PathBuf, ObjectHash)>, filter: Vec, read_content: F, ) -> Vec where - F: Fn(&PathBuf, &SHA1) -> Vec, + F: Fn(&PathBuf, &ObjectHash) -> Vec, { let (processed_files, old_blobs_map, new_blobs_map) = Self::prepare_diff_data(old_blobs, new_blobs, &filter); @@ -148,13 +148,16 @@ impl Diff { /// Build maps, union file set, and apply filter/path checks fn prepare_diff_data( - old_blobs: Vec<(PathBuf, SHA1)>, - new_blobs: Vec<(PathBuf, SHA1)>, + old_blobs: Vec<(PathBuf, ObjectHash)>, + new_blobs: Vec<(PathBuf, ObjectHash)>, filter: &[PathBuf], - ) -> (Vec, HashMap, HashMap) { - let old_blobs_map: HashMap = old_blobs.into_iter().collect(); - let new_blobs_map: HashMap = new_blobs.into_iter().collect(); - + ) -> ( + Vec, + HashMap, + HashMap, + ) { + let old_blobs_map: HashMap = old_blobs.into_iter().collect(); + let new_blobs_map: HashMap = new_blobs.into_iter().collect(); // union set let union_files: HashSet = old_blobs_map .keys() @@ -174,8 +177,8 @@ impl Diff { fn should_process( file: &PathBuf, filter: &[PathBuf], - old_blobs: &HashMap, - new_blobs: &HashMap, + old_blobs: &HashMap, + new_blobs: &HashMap, ) -> bool { if !filter.is_empty() && !filter @@ -194,7 +197,7 @@ impl Diff { Ok(path_abs.starts_with(parent_abs)) } - fn short_hash(hash: Option<&SHA1>) -> String { + fn short_hash(hash: Option<&ObjectHash>) -> String { hash.map(|h| { let hex = h.to_string(); let take = Self::SHORT_HASH_LEN.min(hex.len()); @@ -206,9 +209,9 @@ impl Diff { /// Format a single file's unified diff string. pub fn diff_for_file_string( file: &PathBuf, - old_blobs: &HashMap, - new_blobs: &HashMap, - read_content: &dyn Fn(&PathBuf, &SHA1) -> Vec, + old_blobs: &HashMap, + new_blobs: &HashMap, + read_content: &dyn Fn(&PathBuf, &ObjectHash) -> Vec, ) -> String { let new_hash = new_blobs.get(file); let old_hash = old_blobs.get(file); @@ -221,8 +224,8 @@ impl Diff { /// Format a single file's unified diff using preloaded bytes to avoid re-reading. fn diff_for_file_preloaded( file: &Path, - old_hash: Option<&SHA1>, - new_hash: Option<&SHA1>, + old_hash: Option<&ObjectHash>, + new_hash: Option<&ObjectHash>, old_bytes: &[u8], new_bytes: &[u8], ) -> String { @@ -478,19 +481,23 @@ pub fn compute_diff(old_lines: &[String], new_lines: &[String]) -> Vec (String, SHA1, SHA1) { + fn run_diff( + logical_path: &str, + old_bytes: &[u8], + new_bytes: &[u8], + ) -> (String, ObjectHash, ObjectHash) { let file = PathBuf::from(logical_path); - let old_hash = SHA1::new(old_bytes); - let new_hash = SHA1::new(new_bytes); + let old_hash = ObjectHash::new(old_bytes); + let new_hash = ObjectHash::new(new_bytes); - let mut blob_store: HashMap> = HashMap::new(); + let mut blob_store: HashMap> = HashMap::new(); blob_store.insert(old_hash, old_bytes.to_vec()); blob_store.insert(new_hash, new_bytes.to_vec()); @@ -499,14 +506,15 @@ mod tests { old_map.insert(file.clone(), old_hash); new_map.insert(file.clone(), new_hash); - let reader = - |_: &PathBuf, h: &SHA1| -> Vec { blob_store.get(h).cloned().unwrap_or_default() }; + let reader = |_: &PathBuf, h: &ObjectHash| -> Vec { + blob_store.get(h).cloned().unwrap_or_default() + }; let diff = Diff::diff_for_file_string(&file, &old_map, &new_map, &reader); (diff, old_hash, new_hash) } - fn short_hash(hash: &SHA1) -> String { + fn short_hash(hash: &ObjectHash) -> String { hash.to_string().chars().take(7).collect() } @@ -514,8 +522,8 @@ mod tests { logical_path: &str, old_bytes: &[u8], new_bytes: &[u8], - old_hash: &SHA1, - new_hash: &SHA1, + old_hash: &ObjectHash, + new_hash: &ObjectHash, ) -> Option { let temp_dir = tempdir().ok()?; let old_file = temp_dir.path().join("old.txt"); @@ -564,6 +572,7 @@ mod tests { #[test] fn unified_diff_basic_changes() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old = b"a\nb\nc\n" as &[u8]; let new = b"a\nB\nc\nd\n" as &[u8]; let (diff, _, _) = run_diff("foo.txt", old, new); @@ -580,6 +589,7 @@ mod tests { #[test] fn binary_files_detection() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old_bytes = vec![0u8, 159, 146, 150]; let new_bytes = vec![0xFF, 0x00, 0x01]; let (diff, _, _) = run_diff("bin.dat", &old_bytes, &new_bytes); @@ -588,6 +598,7 @@ mod tests { #[test] fn diff_matches_git_for_fixture() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); //use it to test SHA1/SHA-256 diffs as well let base: PathBuf = [env!("CARGO_MANIFEST_DIR"), "tests", "diff"] .iter() .collect(); @@ -628,6 +639,7 @@ mod tests { #[test] fn diff_matches_git_for_large_change() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old_lines: Vec = (0..5_000).map(|i| format!("line {i}")).collect(); let mut new_lines = old_lines.clone(); for idx in [10, 499, 1_234, 3_210, 4_999] { @@ -670,6 +682,7 @@ mod tests { #[test] fn compute_diff_operations_basic_mapping() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); let old_lines = vec!["a".to_string(), "b".to_string(), "c".to_string()]; let new_lines = vec![ "a".to_string(), diff --git a/src/internal/index.rs b/src/internal/index.rs index 40b7e170..a74eb897 100644 --- a/src/internal/index.rs +++ b/src/internal/index.rs @@ -12,46 +12,9 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use sha1::{Digest, Sha1}; use crate::errors::GitError; -use crate::hash::{ObjectHash, get_hash_kind, HashKind}; +use crate::hash::{ObjectHash, get_hash_kind}; use crate::internal::pack::wrapper::Wrapper; -use crate::utils; - -/// Different hash algorithm enum -#[derive(Clone)] -enum Hashalgorithm { - Sha1(Sha1), - Sha256(sha2::Sha256), - // Future: support other hash algorithms -} -impl Hashalgorithm { - /// Update hash with data - fn update(&mut self, data: &[u8]) { - match self { - Hashalgorithm::Sha1(hasher) => hasher.update(data), - Hashalgorithm::Sha256(hasher) => hasher.update(data), - } - } - /// Finalize and get hash result - fn finalize(self) -> Vec { - match self { - Hashalgorithm::Sha1(hasher) => hasher.finalize().to_vec(), - Hashalgorithm::Sha256(hasher) => hasher.finalize().to_vec(), - } - } - fn new() -> Self { - match get_hash_kind() { - HashKind::Sha1 => Hashalgorithm::Sha1(Sha1::new()), - HashKind::Sha256 => Hashalgorithm::Sha256(sha2::Sha256::new()), - } - } -} -impl std::io::Write for Hashalgorithm { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.update(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> io::Result<()> { Ok(()) } -} +use crate::utils::{self, Hashalgorithm}; #[derive(PartialEq, Eq, Debug, Clone)] pub struct Time { @@ -579,7 +542,7 @@ impl Index { #[cfg(test)] mod tests { use super::*; - use crate::hash::{set_hash_kind_for_test,HashKind}; + use crate::hash::{HashKind, set_hash_kind_for_test}; #[test] fn test_time() { let time = Time { @@ -619,7 +582,7 @@ mod tests { let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("tests/data/index/index-9-256"); - let index = Index::from_file(source).unwrap();//index-里有文件名不是 UTF‑8,Index::from_file 里用 String::from_utf8(name)? 直接就报错了 + let index = Index::from_file(source).unwrap(); assert_eq!(index.size(), 9); for (_, entry) in index.entries.iter() { println!("{entry}"); @@ -639,6 +602,7 @@ mod tests { #[test] fn test_index_entry_create() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); source.push("Cargo.toml"); @@ -648,4 +612,16 @@ mod tests { let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); println!("{entry}"); } + #[test] + fn test_index_entry_create_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); + let mut source = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + source.push("Cargo.toml"); + + let file = Path::new(source.as_path()); + let hash = ObjectHash::from_bytes(&[0u8; 32]).unwrap(); + let workdir = Path::new("../"); + let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); + println!("{entry}"); + } } diff --git a/src/internal/pack/encode.rs b/src/internal/pack/encode.rs index a73d0edf..e01a21ae 100644 --- a/src/internal/pack/encode.rs +++ b/src/internal/pack/encode.rs @@ -8,15 +8,13 @@ use crate::internal::object::types::ObjectType; use crate::time_it; use crate::zstdelta; use crate::{ - errors::GitError, - hash::{HashKind, ObjectHash, get_hash_kind}, - internal::pack::entry::Entry, + errors::GitError, hash::ObjectHash, internal::pack::entry::Entry, utils::Hashalgorithm, }; use ahash::AHasher; use flate2::write::ZlibEncoder; use natord::compare; use rayon::prelude::*; -use sha1::{Digest, Sha1}; + use std::hash::{Hash, Hasher}; use std::path::Path; use tokio::sync::mpsc; @@ -26,35 +24,6 @@ const MAX_CHAIN_LEN: usize = 50; const MIN_DELTA_RATE: f64 = 0.5; // minimum delta rate //const MAX_ZSTDELTA_CHAIN_LEN: usize = 50; -/// Different hash algorithm enum -#[derive(Clone)] -enum Hashalgorithm { - Sha1(Sha1), - Sha256(sha2::Sha256), - // Future: support other hash algorithms -} -impl Hashalgorithm { - /// Update hash with data - fn update(&mut self, data: &[u8]) { - match self { - Hashalgorithm::Sha1(hasher) => hasher.update(data), - Hashalgorithm::Sha256(hasher) => hasher.update(data), - } - } - /// Finalize and get hash result - fn finalize(self) -> Vec { - match self { - Hashalgorithm::Sha1(hasher) => hasher.finalize().to_vec(), - Hashalgorithm::Sha256(hasher) => hasher.finalize().to_vec(), - } - } - fn new() -> Self { - match get_hash_kind() { - HashKind::Sha1 => Hashalgorithm::Sha1(Sha1::new()), - HashKind::Sha256 => Hashalgorithm::Sha256(sha2::Sha256::new()), - } - } -} /// A encoder for generating pack files with delta objects. pub struct PackEncoder { object_number: usize, diff --git a/src/utils.rs b/src/utils.rs index b6afcfc8..46d24000 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,7 +1,7 @@ -use crate::hash::ObjectHash; +use crate::hash::{HashKind, ObjectHash, get_hash_kind}; +use sha1::{Digest, Sha1}; use std::io; use std::io::{BufRead, Read}; - pub fn read_bytes(file: &mut impl Read, len: usize) -> io::Result> { let mut buf = vec![0; len]; file.read_exact(&mut buf)?; @@ -47,3 +47,41 @@ impl BufRead for CountingReader { self.inner.consume(amt); } } +/// Different hash algorithm enum +#[derive(Clone)] +pub enum Hashalgorithm { + Sha1(Sha1), + Sha256(sha2::Sha256), + // Future: support other hash algorithms +} +impl Hashalgorithm { + /// Update hash with data + pub fn update(&mut self, data: &[u8]) { + match self { + Hashalgorithm::Sha1(hasher) => hasher.update(data), + Hashalgorithm::Sha256(hasher) => hasher.update(data), + } + } + /// Finalize and get hash result + pub fn finalize(self) -> Vec { + match self { + Hashalgorithm::Sha1(hasher) => hasher.finalize().to_vec(), + Hashalgorithm::Sha256(hasher) => hasher.finalize().to_vec(), + } + } + pub fn new() -> Self { + match get_hash_kind() { + HashKind::Sha1 => Hashalgorithm::Sha1(Sha1::new()), + HashKind::Sha256 => Hashalgorithm::Sha256(sha2::Sha256::new()), + } + } +} +impl std::io::Write for Hashalgorithm { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.update(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} From ecf01567530569ebd1c28ec1fd58e05978a4b7f8 Mon Sep 17 00:00:00 2001 From: jackieismpc Date: Thu, 27 Nov 2025 21:18:37 +0800 Subject: [PATCH 6/7] 1.Complete the remaining changes. 2.The underlying smart.rs layer already supports negotiating different hash algorithms. Signed-off-by: jackieismpc --- src/hash.rs | 12 +++++- src/protocol/core.rs | 8 ++-- src/protocol/pack.rs | 97 ++++++++++++++++++++++++++++++++++++++++++- src/protocol/smart.rs | 86 +++++++++++++++++++++++++++++++------- src/protocol/types.rs | 3 -- 5 files changed, 181 insertions(+), 25 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 8389d9d3..944eeda8 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -3,7 +3,7 @@ //! location in the Git internal and mega database. //! -use std::{cell::RefCell, fmt::Display, io, str::FromStr}; +use std::{cell::RefCell, fmt::Display, hash::Hash, io, str::FromStr}; use crate::internal::object::types::ObjectType; use bincode::{Decode, Encode}; @@ -145,6 +145,16 @@ impl FromStr for ObjectHash { /// - `_to_string`: Converts the hash to a hexadecimal string representation. /// impl ObjectHash { + /// returns a zeroed hash value for the given hash kind + pub fn zero_str(kind: HashKind) -> String { + match kind { + HashKind::Sha1 => "0000000000000000000000000000000000000000".to_string(), + HashKind::Sha256 => { + "0000000000000000000000000000000000000000000000000000000000000000".to_string() + } + } + } + /// returns the kind of hash pub fn kind(&self) -> HashKind { match self { diff --git a/src/protocol/core.rs b/src/protocol/core.rs index 7bdf0a63..2bfe2b27 100644 --- a/src/protocol/core.rs +++ b/src/protocol/core.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use bytes::Bytes; use futures::stream::StreamExt; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::object::ObjectTrait; use crate::protocol::smart::SmartProtocol; @@ -63,7 +63,7 @@ pub trait RepositoryAccess: Send + Sync + Clone { object_hash: &str, ) -> Result { let data = self.get_object(object_hash).await?; - let hash = SHA1::from_str(object_hash) + let hash = ObjectHash::from_str(object_hash) .map_err(|e| ProtocolError::repository_error(format!("Invalid hash format: {}", e)))?; crate::internal::object::blob::Blob::from_bytes(&data, hash) @@ -79,7 +79,7 @@ pub trait RepositoryAccess: Send + Sync + Clone { commit_hash: &str, ) -> Result { let data = self.get_object(commit_hash).await?; - let hash = SHA1::from_str(commit_hash) + let hash = ObjectHash::from_str(commit_hash) .map_err(|e| ProtocolError::repository_error(format!("Invalid hash format: {}", e)))?; crate::internal::object::commit::Commit::from_bytes(&data, hash) @@ -95,7 +95,7 @@ pub trait RepositoryAccess: Send + Sync + Clone { tree_hash: &str, ) -> Result { let data = self.get_object(tree_hash).await?; - let hash = SHA1::from_str(tree_hash) + let hash = ObjectHash::from_str(tree_hash) .map_err(|e| ProtocolError::repository_error(format!("Invalid hash format: {}", e)))?; crate::internal::object::tree::Tree::from_bytes(&data, hash) diff --git a/src/protocol/pack.rs b/src/protocol/pack.rs index 3083d530..e6f5e358 100644 --- a/src/protocol/pack.rs +++ b/src/protocol/pack.rs @@ -1,6 +1,6 @@ use super::core::RepositoryAccess; use super::types::ProtocolError; -use crate::hash::SHA1; +use crate::hash::ObjectHash; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::object::types::ObjectType; use crate::internal::object::{ObjectTrait, blob::Blob, commit::Commit, tree::Tree}; @@ -126,7 +126,7 @@ where tracing::warn!("Unknown object type in pack: {:?}", entry.inner.obj_type); } }, - None::, + None::, ) .map_err(|e| ProtocolError::invalid_request(&format!("Failed to decode pack: {}", e)))?; @@ -348,6 +348,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; use crate::internal::object::blob::Blob; use crate::internal::object::commit::Commit; use crate::internal::object::signature::{Signature, SignatureType}; @@ -399,6 +400,98 @@ mod tests { #[tokio::test] async fn test_pack_roundtrip_encode_decode() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); + // Create two Blob objects + let blob1 = Blob::from_content("hello"); + let blob2 = Blob::from_content("world"); + + // Create a Tree containing two file items + let item1 = TreeItem::new(TreeItemMode::Blob, blob1.id, "hello.txt".to_string()); + let item2 = TreeItem::new(TreeItemMode::Blob, blob2.id, "world.txt".to_string()); + let tree = Tree::from_tree_items(vec![item1, item2]).unwrap(); + + // Create a Commit pointing to the Tree + let author = Signature::new( + SignatureType::Author, + "tester".to_string(), + "tester@example.com".to_string(), + ); + let committer = Signature::new( + SignatureType::Committer, + "tester".to_string(), + "tester@example.com".to_string(), + ); + let commit = Commit::new(author, committer, tree.id, vec![], "init commit"); + + // Generate pack stream + let (tx, mut rx) = mpsc::channel::>(64); + PackGenerator::::generate_pack_stream( + ( + vec![commit.clone()], + vec![tree.clone()], + vec![blob1.clone(), blob2.clone()], + ), + tx, + ) + .await + .unwrap(); + + let mut pack_bytes: Vec = Vec::new(); + while let Some(chunk) = rx.recv().await { + pack_bytes.extend_from_slice(&chunk); + } + println!("Encoded pack size: {} bytes", pack_bytes.len()); + + // Unpack the pack stream + let dummy = DummyRepoAccess; + let generator = PackGenerator::new(&dummy); + let (decoded_commits, decoded_trees, decoded_blobs) = generator + .unpack_stream(Bytes::from(pack_bytes)) + .await + .unwrap(); + + println!( + "Decoded commits: {:?}", + decoded_commits + .iter() + .map(|c| c.id.to_string()) + .collect::>() + ); + println!( + "Decoded trees: {:?}", + decoded_trees + .iter() + .map(|t| t.id.to_string()) + .collect::>() + ); + println!( + "Decoded blobs: {:?}", + decoded_blobs + .iter() + .map(|b| b.id.to_string()) + .collect::>() + ); + + // Verify object ID roundtrip consistency + assert_eq!(decoded_commits.len(), 1); + assert_eq!(decoded_trees.len(), 1); + assert_eq!(decoded_blobs.len(), 2); + + assert_eq!(decoded_commits[0].id, commit.id); + assert_eq!(decoded_trees[0].id, tree.id); + + let mut orig_blob_ids = vec![blob1.id.to_string(), blob2.id.to_string()]; + orig_blob_ids.sort(); + let mut decoded_blob_ids = decoded_blobs + .iter() + .map(|b| b.id.to_string()) + .collect::>(); + decoded_blob_ids.sort(); + assert_eq!(orig_blob_ids, decoded_blob_ids); + } + #[tokio::test] + async fn test_pack_roundtrip_encode_decode_sha256() { + let _guard = set_hash_kind_for_test(HashKind::Sha256); // Create two Blob objects let blob1 = Blob::from_content("hello"); let blob2 = Blob::from_content("world"); diff --git a/src/protocol/smart.rs b/src/protocol/smart.rs index 546a63af..69ea1ed4 100644 --- a/src/protocol/smart.rs +++ b/src/protocol/smart.rs @@ -8,10 +8,9 @@ use super::types::ProtocolError; use super::types::{ COMMON_CAP_LIST, Capability, LF, NUL, PKT_LINE_END_MARKER, ProtocolStream, RECEIVE_CAP_LIST, RefCommand, RefTypeEnum, SP, ServiceType, SideBand, TransportProtocol, UPLOAD_CAP_LIST, - ZERO_ID, }; use super::utils::{add_pkt_line_string, build_smart_reply, read_pkt_line, read_until_white_space}; - +use crate::hash::{HashKind, ObjectHash, get_hash_kind}; /// Smart Git Protocol implementation /// /// This struct handles the Git smart protocol operations for both HTTP and SSH transports. @@ -25,7 +24,9 @@ where pub capabilities: Vec, pub side_band: Option, pub command_list: Vec, - + pub wire_hash_kind: HashKind, + pub local_hash_kind: HashKind, + pub zero_id: String, // Trait-based dependencies repo_storage: R, auth_service: A, @@ -36,6 +37,11 @@ where R: RepositoryAccess, A: AuthenticationService, { + pub fn set_wire_hash_kind(&mut self, kind: HashKind) { + self.wire_hash_kind = kind; + self.zero_id = ObjectHash::zero_str(kind); + } + /// Create a new SmartProtocol instance pub fn new(transport_protocol: TransportProtocol, repo_storage: R, auth_service: A) -> Self { Self { @@ -45,6 +51,9 @@ where command_list: Vec::new(), repo_storage, auth_service, + wire_hash_kind: HashKind::default(), // Default to SHA-1 + local_hash_kind: get_hash_kind(), + zero_id: ObjectHash::zero_str(HashKind::default()), } } @@ -81,7 +90,17 @@ where self.repo_storage.get_repository_refs().await.map_err(|e| { ProtocolError::repository_error(format!("Failed to get refs: {}", e)) })?; - + let hex_len = self.wire_hash_kind.hex_len(); + for (name, h) in &refs { + if h.len() != hex_len { + return Err(ProtocolError::invalid_request(&format!( + "Hash length mismatch for ref {}: expected {}, got {}", + name, + hex_len, + h.len() + ))); + } + } // Ensure refs match the expected wire hash kind // Convert to the expected format (head_hash, git_refs) let head_hash = refs .iter() @@ -89,21 +108,25 @@ where name == "HEAD" || name.ends_with("/main") || name.ends_with("/master") }) .map(|(_, hash)| hash.clone()) - .unwrap_or_else(|| "0000000000000000000000000000000000000000".to_string()); + .unwrap_or_else(|| self.zero_id.clone()); let git_refs: Vec = refs .into_iter() .map(|(name, hash)| super::types::GitRef { name, hash }) .collect(); - + // capability add object-format,declare the wire hash kind + let format_cap = match self.wire_hash_kind { + HashKind::Sha1 => " object-format=sha1", + HashKind::Sha256 => " object-format=sha256", + }; // Determine capabilities based on service type let cap_list = match service_type { - ServiceType::UploadPack => format!("{UPLOAD_CAP_LIST}{COMMON_CAP_LIST}"), - ServiceType::ReceivePack => format!("{RECEIVE_CAP_LIST}{COMMON_CAP_LIST}"), + ServiceType::UploadPack => format!("{UPLOAD_CAP_LIST}{COMMON_CAP_LIST}{format_cap}"), + ServiceType::ReceivePack => format!("{RECEIVE_CAP_LIST}{COMMON_CAP_LIST}{format_cap}"), }; // The stream MUST include capability declarations behind a NUL on the first ref. - let name = if head_hash == ZERO_ID { + let name = if head_hash == self.zero_id { "capabilities^{}" } else { "HEAD" @@ -275,8 +298,8 @@ where for command in &mut self.command_list { if command.ref_type == RefTypeEnum::Tag { // Just update if refs type is tag - // Convert ZERO_ID to None for old hash - let old_hash = if command.old_hash == ZERO_ID { + // Convert ze to None for old hash + let old_hash = if command.old_hash == self.zero_id { None } else { Some(command.old_hash.as_str()) @@ -294,8 +317,8 @@ where command.default_branch = true; default_exist = true; } - // Convert ZERO_ID to None for old hash - let old_hash = if command.old_hash == ZERO_ID { + // Convert zero_id to None for old hash + let old_hash = if command.old_hash == self.zero_id { None } else { Some(command.old_hash.as_str()) @@ -339,6 +362,16 @@ where /// Parse capabilities from capability string pub fn parse_capabilities(&mut self, cap_str: &str) { for cap in cap_str.split_whitespace() { + if let Some(fmt) = cap.strip_prefix("object-format=") { + match fmt { + "sha1" => self.set_wire_hash_kind(HashKind::Sha1), + "sha256" => self.set_wire_hash_kind(HashKind::Sha256), + _ => { + tracing::warn!("Unknown object-format capability: {}", fmt); + } + } + continue; + } if let Ok(capability) = cap.parse::() { self.capabilities.push(capability); } @@ -359,13 +392,14 @@ where #[cfg(test)] mod tests { use super::*; + use crate::hash::{HashKind, set_hash_kind_for_test}; use crate::internal::metadata::{EntryMeta, MetaAttached}; use crate::internal::object::blob::Blob; use crate::internal::object::commit::Commit; use crate::internal::object::signature::{Signature, SignatureType}; use crate::internal::object::tree::{Tree, TreeItem, TreeItemMode}; use crate::internal::pack::{encode::PackEncoder, entry::Entry}; - use crate::protocol::types::{RefCommand, ZERO_ID}; // import sibling types + use crate::protocol::types::RefCommand; // import sibling types use crate::protocol::utils; // import sibling module use async_trait::async_trait; use bytes::Bytes; @@ -493,6 +527,7 @@ mod tests { #[tokio::test] async fn test_receive_pack_stream_status_report() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); // Build simple objects let blob1 = Blob::from_content("hello"); let blob2 = Blob::from_content("world"); @@ -565,8 +600,9 @@ mod tests { let repo_access = TestRepoAccess::new(); let auth = TestAuth; let mut smart = SmartProtocol::new(TransportProtocol::Http, repo_access.clone(), auth); + smart.set_wire_hash_kind(HashKind::Sha1); smart.command_list.push(RefCommand::new( - ZERO_ID.to_string(), + smart.zero_id.to_string(), commit.id.to_string(), "refs/heads/main".to_string(), )); @@ -599,4 +635,24 @@ mod tests { assert_eq!(repo_access.updates_len(), 1); assert!(repo_access.post_hook_called()); } + + #[tokio::test] + async fn info_refs_rejects_sha256_with_sha1_refs() { + let _guard = set_hash_kind_for_test(HashKind::Sha1); // avoid thread-local contamination + let repo_access = TestRepoAccess::new(); // still returns 40-char strings + let auth = TestAuth; + let mut smart = SmartProtocol::new(TransportProtocol::Http, repo_access, auth); + smart.set_wire_hash_kind(HashKind::Sha256); // claims wire uses SHA-256 + // expect failure because refs are SHA-1 + let res = smart.git_info_refs(ServiceType::UploadPack).await; + assert!(res.is_err(), "expected failure when hash lengths mismatch"); + + smart.set_wire_hash_kind(HashKind::Sha1); + + let res = smart.git_info_refs(ServiceType::UploadPack).await; + assert!( + res.is_ok(), + "expected SHA1 refs to be accepted when wire is SHA1" + ); + } } diff --git a/src/protocol/types.rs b/src/protocol/types.rs index 14a8f5b1..fb62c674 100644 --- a/src/protocol/types.rs +++ b/src/protocol/types.rs @@ -364,9 +364,6 @@ pub enum CommandType { Delete, } -/// Zero object ID constant -pub const ZERO_ID: &str = "0000000000000000000000000000000000000000"; - /// Protocol constants pub const LF: char = '\n'; pub const SP: char = ' '; From b480c4ff6a5f82379c8415a786a0a797c4369270 Mon Sep 17 00:00:00 2001 From: jackieismpc Date: Fri, 28 Nov 2025 09:53:14 +0800 Subject: [PATCH 7/7] [ci] Fix CI issues for multi-hash support Signed-off-by: jackieismpc --- BUCK | 1 + src/internal/index.rs | 26 ++++++++++++++------------ src/internal/object/commit.rs | 2 +- src/internal/object/tree.rs | 4 ++-- src/internal/pack/cache.rs | 2 +- src/internal/pack/cache_object.rs | 4 ++-- src/internal/pack/decode.rs | 12 ++++++------ src/internal/pack/encode.rs | 8 ++++---- src/internal/pack/wrapper.rs | 26 ++++++++++---------------- src/protocol/smart.rs | 2 +- src/utils.rs | 27 +++++++++++++++++---------- 11 files changed, 59 insertions(+), 55 deletions(-) diff --git a/BUCK b/BUCK index d25eb0e4..959c3dd9 100644 --- a/BUCK +++ b/BUCK @@ -73,5 +73,6 @@ rust_library( "//third-party/rust/crates/tracing/0.1.41:tracing", "//third-party/rust/crates/uuid/1.18.1:uuid", "//third-party/rust/crates/zstd-sys/2.0.16+zstd.1.5.7:zstd-sys", + "//third-party/rust/crates/sha2/0.10.9:sha2", ], ) diff --git a/src/internal/index.rs b/src/internal/index.rs index a74eb897..0ab41b19 100644 --- a/src/internal/index.rs +++ b/src/internal/index.rs @@ -9,12 +9,11 @@ use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; -use sha1::{Digest, Sha1}; use crate::errors::GitError; use crate::hash::{ObjectHash, get_hash_kind}; use crate::internal::pack::wrapper::Wrapper; -use crate::utils::{self, Hashalgorithm}; +use crate::utils::{self, HashAlgorithm}; #[derive(PartialEq, Eq, Debug, Clone)] pub struct Time { @@ -281,15 +280,16 @@ impl Index { .insert((entry.name.clone(), entry.flags.stage), entry); // 1-8 nul bytes as necessary to pad the entry to a multiple of eight bytes - // while keeping the name NUL-terminated. // so at least 1 byte nul + // while keeping the name NUL-terminated. let hash_len = get_hash_kind().size(); - let padding = (8 - ((hash_len + 2 + name_len) % 8)) % 8; // 2 for flags + let entry_len = hash_len + 2 + name_len; + let padding = 1 + ((8 - ((entry_len + 1) % 8)) % 8); // at least 1 byte nul utils::read_bytes(file, padding)?; } // Extensions while file.bytes_read() + get_hash_kind().size() < total_size as usize { - // The remaining 20 bytes must be checksum + // The remaining bytes must be the pack checksum (size = get_hash_kind().size()) let sign = utils::read_bytes(file, 4)?; println!( "{:?}", @@ -321,7 +321,7 @@ impl Index { pub fn to_file(&self, path: impl AsRef) -> Result<(), GitError> { let mut file = File::create(path)?; - let mut hash = Sha1::new(); + let mut hash = HashAlgorithm::new(); let mut header = Vec::new(); header.write_all(b"DIRC")?; @@ -342,12 +342,13 @@ impl Index { entry_bytes.write_u32::(entry.uid)?; entry_bytes.write_u32::(entry.gid)?; entry_bytes.write_u32::(entry.size)?; - entry_bytes.write_all(&entry.hash.as_ref())?; + entry_bytes.write_all(entry.hash.as_ref())?; entry_bytes.write_u16::((&entry.flags).try_into().unwrap())?; entry_bytes.write_all(entry.name.as_bytes())?; - let padding = 8 - ((22 + entry.name.len()) % 8); + let hash_len = get_hash_kind().size(); + let entry_len = hash_len + 2 + entry.name.len(); + let padding = 1 + ((8 - ((entry_len + 1) % 8)) % 8); // at least 1 byte nul entry_bytes.write_all(&vec![0; padding])?; - file.write_all(&entry_bytes)?; hash.update(&entry_bytes); } @@ -355,8 +356,9 @@ impl Index { // Extensions // check sum - let file_hash: [u8; 20] = hash.finalize().into(); - file.write_all(&file_hash)?; + let file_hash = + ObjectHash::from_bytes(&hash.finalize()).map_err(GitError::InvalidIndexFile)?; + file.write_all(file_hash.as_ref())?; Ok(()) } @@ -384,7 +386,7 @@ impl Index { // re-calculate SHA1/SHA256 let mut file = File::open(&abs_path)?; - let mut hasher = Hashalgorithm::new(); + let mut hasher = HashAlgorithm::new(); io::copy(&mut file, &mut hasher)?; let new_hash = ObjectHash::from_bytes(&hasher.finalize()).unwrap(); diff --git a/src/internal/object/commit.rs b/src/internal/object/commit.rs index 679f65de..40eb470a 100644 --- a/src/internal/object/commit.rs +++ b/src/internal/object/commit.rs @@ -360,7 +360,7 @@ signed sha256 commit for test"#; assert_eq!(commit.author.name, "jackieismpc"); assert_eq!(commit.author.email, "jackieismpc@gmail.com"); assert_eq!(commit.committer.name, "jackieismpc"); - // check message content(must contains gpgsig-sha256 and content) + // // check message content (must contain gpgsig-sha256 and content) assert!(commit.message.contains("-----BEGIN PGP SIGNATURE-----")); assert!(commit.message.contains("-----END PGP SIGNATURE-----")); assert!(commit.message.contains("signed sha256 commit for test")); diff --git a/src/internal/object/tree.rs b/src/internal/object/tree.rs index 13204c79..94d7a2f8 100644 --- a/src/internal/object/tree.rs +++ b/src/internal/object/tree.rs @@ -413,7 +413,7 @@ mod tests { let bytes = tree_item.to_data(); - // 一个小 helper:把十六进制字符串转成字节序列 + // a small helper: convert hex string to byte sequence fn hex_to_bytes(s: &str) -> Vec { assert!(s.len() % 2 == 0); let mut out = Vec::with_capacity(s.len() / 2); @@ -424,7 +424,7 @@ mod tests { out } - // 期望的编码:`"100644 hello-world\0" + <32字节的blob哈希>` + // expected encoding: `"100644 hello-world\0" + <32-byte blob hash>` let mut expected = b"100644 hello-world\0".to_vec(); expected.extend_from_slice(&hex_to_bytes( "2cf8d83d9ee29543b34a87727421fdecb7e3f3a183d337639025de576db9ebb4", diff --git a/src/internal/pack/cache.rs b/src/internal/pack/cache.rs index 08309112..ed422af3 100644 --- a/src/internal/pack/cache.rs +++ b/src/internal/pack/cache.rs @@ -89,7 +89,7 @@ impl Caches { /// generate the temp file path, hex string of the hash fn generate_temp_path(&self, tmp_path: &Path, hash: ObjectHash) -> PathBuf { - // This is enough for the original path, 2 chars directory, 40/64 chars hash, and extra slashes + // Reserve capacity for base path, 2-char subdir, hex hash string, and separators let mut path = PathBuf::with_capacity(self.tmp_path.capacity() + hash.to_string().len() + 5); path.push(tmp_path); diff --git a/src/internal/pack/cache_object.rs b/src/internal/pack/cache_object.rs index b65975e2..52a22eb7 100644 --- a/src/internal/pack/cache_object.rs +++ b/src/internal/pack/cache_object.rs @@ -478,11 +478,11 @@ mod test { ); assert!(r.is_err()); if let Err(lru_mem::TryInsertError::WouldEjectLru { .. }) = r { - // 匹配到指定错误,不需要额外操作 + // match the specified error, no extra action needed } else { panic!("Expected WouldEjectLru error"); } - // 使用不同的键插入b,这样a会被驱逐 + // insert b with a different key, so a will be ejected let r = cache.insert( hash_b.to_string(), ArcWrapper::new(Arc::new(b.clone()), Arc::new(AtomicBool::new(true)), None), diff --git a/src/internal/pack/decode.rs b/src/internal/pack/decode.rs index 5864c4e9..92bcf666 100644 --- a/src/internal/pack/decode.rs +++ b/src/internal/pack/decode.rs @@ -309,8 +309,8 @@ impl Pack { }) } ObjectType::HashDelta => { - // Read 20/32 bytes to get the reference object SHA1/SHA256 hash - let ref_sha1 = ObjectHash::from_stream(pack).unwrap(); + // Read hash bytes to get the reference object hash(size depends on hash kind,e.g.,20 for SHA1,32 for SHA256) + let ref_sha = ObjectHash::from_stream(pack).unwrap(); // Offset is incremented by 20/32 bytes *offset += get_hash_kind().size(); @@ -321,7 +321,7 @@ impl Pack { let (_, final_size) = utils::read_delta_object_size(&mut reader)?; Ok(CacheObject { - info: CacheObjectInfo::HashDelta(ref_sha1, final_size), + info: CacheObjectInfo::HashDelta(ref_sha, final_size), offset: init_offset, data_decompressed: data, mem_recorder: None, @@ -881,7 +881,7 @@ mod tests { let mut buffered = BufReader::new(f); let mut p = Pack::new( Some(20), - Some(1024 * 1024 * 1024 * 2), + Some(1024 * 1024 * 1024 * 1), //try to avoid dead lock on CI servers with low memory Some(tmp.clone()), true, ); @@ -933,7 +933,7 @@ mod tests { let stream = ReaderStream::new(f).map_err(axum::Error::new); let p = Pack::new( Some(20), - Some(1024 * 1024 * 1024 * 4), + Some(1024 * 1024 * 1024 * 1), Some(tmp.clone()), true, ); @@ -997,7 +997,7 @@ mod tests { let buffered = BufReader::new(f); let p = Pack::new( Some(20), - Some(1024 * 1024 * 1024 * 2), + Some(1024 * 1024 * 1024 * 1), Some(tmp.clone()), true, ); diff --git a/src/internal/pack/encode.rs b/src/internal/pack/encode.rs index e01a21ae..43882528 100644 --- a/src/internal/pack/encode.rs +++ b/src/internal/pack/encode.rs @@ -8,7 +8,7 @@ use crate::internal::object::types::ObjectType; use crate::time_it; use crate::zstdelta; use crate::{ - errors::GitError, hash::ObjectHash, internal::pack::entry::Entry, utils::Hashalgorithm, + errors::GitError, hash::ObjectHash, internal::pack::entry::Entry, utils::HashAlgorithm, }; use ahash::AHasher; use flate2::write::ZlibEncoder; @@ -32,7 +32,7 @@ pub struct PackEncoder { // window: VecDeque<(Entry, usize)>, // entry and offset sender: Option>>, inner_offset: usize, // offset of current entry - inner_hash: Hashalgorithm, // introduce different hash algorithm + inner_hash: HashAlgorithm, // introduce different hash algorithm final_hash: Option, start_encoding: bool, } @@ -186,8 +186,8 @@ impl PackEncoder { process_index: 0, // window: VecDeque::with_capacity(window_size), sender: Some(sender), - inner_offset: 12, // 12 bytes header - inner_hash: Hashalgorithm::new(), // introduce different hash algorithm + inner_offset: 12, // start after 12 bytes pack header(signature + version + object count). + inner_hash: HashAlgorithm::new(), // introduce different hash algorithm final_hash: None, start_encoding: false, } diff --git a/src/internal/pack/wrapper.rs b/src/internal/pack/wrapper.rs index ba40f315..717cc352 100644 --- a/src/internal/pack/wrapper.rs +++ b/src/internal/pack/wrapper.rs @@ -3,7 +3,7 @@ use std::io::{self, BufRead, Read}; use sha1::{Digest, Sha1}; use crate::hash::{HashKind, ObjectHash, get_hash_kind}; - +use crate::utils::HashAlgorithm; /// [`Wrapper`] is a wrapper around a reader that also computes the SHA1/ SHA256 hash of the data read. /// /// It is designed to work with any reader that implements `BufRead`. @@ -13,15 +13,9 @@ use crate::hash::{HashKind, ObjectHash, get_hash_kind}; /// * `hash`: The hash state. /// * `count_hash`: A flag to indicate whether to compute the hash while reading. /// -/// we abstract the hasher to support both SHA1 and SHA256 Now. -#[derive(Clone)] -enum Hasher { - Sha1(Sha1), - Sha256(sha2::Sha256), -} pub struct Wrapper { inner: R, - hash: Hasher, + hash: HashAlgorithm, bytes_read: usize, } @@ -38,8 +32,8 @@ where Self { inner, hash: match get_hash_kind() { - HashKind::Sha1 => Hasher::Sha1(Sha1::new()), - HashKind::Sha256 => Hasher::Sha256(sha2::Sha256::new()), + HashKind::Sha1 => HashAlgorithm::Sha1(Sha1::new()), + HashKind::Sha256 => HashAlgorithm::Sha256(sha2::Sha256::new()), }, // Initialize a new SHA1/ SHA256 hasher bytes_read: 0, } @@ -54,11 +48,11 @@ where /// This is a clone of the internal hash state finalized into a SHA1/ SHA256 hash. pub fn final_hash(&self) -> ObjectHash { match &self.hash.clone() { - Hasher::Sha1(hasher) => { + HashAlgorithm::Sha1(hasher) => { let re: [u8; 20] = hasher.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes ObjectHash::from_bytes(&re).unwrap() } - Hasher::Sha256(hasher) => { + HashAlgorithm::Sha256(hasher) => { let re: [u8; 32] = hasher.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes ObjectHash::from_bytes(&re).unwrap() } @@ -82,8 +76,8 @@ where fn consume(&mut self, amt: usize) { let buffer = self.inner.fill_buf().expect("Failed to fill buffer"); match &mut self.hash { - Hasher::Sha1(hasher) => hasher.update(&buffer[..amt]), // Update SHA1 hash with the data being consumed - Hasher::Sha256(hasher) => hasher.update(&buffer[..amt]), // Update SHA256 hash with the data being consumed + HashAlgorithm::Sha1(hasher) => hasher.update(&buffer[..amt]), // Update SHA1 hash with the data being consumed + HashAlgorithm::Sha256(hasher) => hasher.update(&buffer[..amt]), // Update SHA256 hash with the data being consumed } self.inner.consume(amt); // Consume the data from the inner reader self.bytes_read += amt; @@ -105,8 +99,8 @@ where fn read(&mut self, buf: &mut [u8]) -> io::Result { let o = self.inner.read(buf)?; // Read data into the buffer match &mut self.hash { - Hasher::Sha1(hasher) => hasher.update(&buf[..o]), // Update SHA1 hash with the data being read - Hasher::Sha256(hasher) => hasher.update(&buf[..o]), // Update SHA256 hash with the data being read + HashAlgorithm::Sha1(hasher) => hasher.update(&buf[..o]), // Update SHA1 hash with the data being read + HashAlgorithm::Sha256(hasher) => hasher.update(&buf[..o]), // Update SHA256 hash with the data being read } self.bytes_read += o; Ok(o) // Return the number of bytes read diff --git a/src/protocol/smart.rs b/src/protocol/smart.rs index 69ea1ed4..0f2855f7 100644 --- a/src/protocol/smart.rs +++ b/src/protocol/smart.rs @@ -298,7 +298,7 @@ where for command in &mut self.command_list { if command.ref_type == RefTypeEnum::Tag { // Just update if refs type is tag - // Convert ze to None for old hash + // Convert zero_id to None for old hash let old_hash = if command.old_hash == self.zero_id { None } else { diff --git a/src/utils.rs b/src/utils.rs index 46d24000..dfaea03c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -47,36 +47,38 @@ impl BufRead for CountingReader { self.inner.consume(amt); } } -/// Different hash algorithm enum +/// a hash abstraction to support both SHA1 and SHA256 +/// which for stream hashing handle use (e.g. Sha1::new()) +/// `std::io::Write` trait to update the hash state #[derive(Clone)] -pub enum Hashalgorithm { +pub enum HashAlgorithm { Sha1(Sha1), Sha256(sha2::Sha256), // Future: support other hash algorithms } -impl Hashalgorithm { +impl HashAlgorithm { /// Update hash with data pub fn update(&mut self, data: &[u8]) { match self { - Hashalgorithm::Sha1(hasher) => hasher.update(data), - Hashalgorithm::Sha256(hasher) => hasher.update(data), + HashAlgorithm::Sha1(hasher) => hasher.update(data), + HashAlgorithm::Sha256(hasher) => hasher.update(data), } } /// Finalize and get hash result pub fn finalize(self) -> Vec { match self { - Hashalgorithm::Sha1(hasher) => hasher.finalize().to_vec(), - Hashalgorithm::Sha256(hasher) => hasher.finalize().to_vec(), + HashAlgorithm::Sha1(hasher) => hasher.finalize().to_vec(), + HashAlgorithm::Sha256(hasher) => hasher.finalize().to_vec(), } } pub fn new() -> Self { match get_hash_kind() { - HashKind::Sha1 => Hashalgorithm::Sha1(Sha1::new()), - HashKind::Sha256 => Hashalgorithm::Sha256(sha2::Sha256::new()), + HashKind::Sha1 => HashAlgorithm::Sha1(Sha1::new()), + HashKind::Sha256 => HashAlgorithm::Sha256(sha2::Sha256::new()), } } } -impl std::io::Write for Hashalgorithm { +impl std::io::Write for HashAlgorithm { fn write(&mut self, buf: &[u8]) -> io::Result { self.update(buf); Ok(buf.len()) @@ -85,3 +87,8 @@ impl std::io::Write for Hashalgorithm { Ok(()) } } +impl Default for HashAlgorithm { + fn default() -> Self { + Self::new() + } +}