diff --git a/Cargo.lock b/Cargo.lock index 213c19e..c29bb8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1173,7 +1173,7 @@ dependencies = [ [[package]] name = "quantitative-data-analysis-rs" -version = "0.38.0" +version = "0.39.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index f91e61d..7a7672c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quantitative-data-analysis-rs" -version = "0.38.0" +version = "0.39.0" edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/infrastructure/dsv.rs b/src/infrastructure/dsv.rs index 61f1b7d..e57e769 100644 --- a/src/infrastructure/dsv.rs +++ b/src/infrastructure/dsv.rs @@ -1,12 +1,20 @@ use bytes::Bytes; +use std::fmt::{Debug, Display}; use std::marker::PhantomData; +use std::sync::{Mutex, OnceLock}; use crate::infrastructure::from_slice::FromSlice; +/// Delimited data parsed lazily and cached per instance after a successful parse. +/// +/// Do not modify `buffer` or `has_headers` after construction. Changes to these +/// public fields do not invalidate the cached items. pub struct Dsv { pub has_headers: bool, pub buffer: Bytes, _structure: PhantomData, + parsed_items: OnceLock>, + initialization: Mutex<()>, } impl Dsv { @@ -15,6 +23,26 @@ impl Dsv { has_headers, buffer, _structure: PhantomData, + parsed_items: OnceLock::new(), + initialization: Mutex::new(()), } } + + pub(crate) fn items(&self) -> anyhow::Result<&[T::Item]> + where + >::Error: Debug + Display, + { + if let Some(items) = self.parsed_items.get() { + return Ok(items); + } + let _guard = self + .initialization + .lock() + .map_err(|error| anyhow::anyhow!("DSV cache initialization lock poisoned: {error}"))?; + if let Some(items) = self.parsed_items.get() { + return Ok(items); + } + let items = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?; + Ok(self.parsed_items.get_or_init(|| items)) + } } diff --git a/src/infrastructure/repositories/company/dsv.rs b/src/infrastructure/repositories/company/dsv.rs index 3dbfdd2..919c1c0 100644 --- a/src/infrastructure/repositories/company/dsv.rs +++ b/src/infrastructure/repositories/company/dsv.rs @@ -20,8 +20,7 @@ where &self, query: &queries::get_company::Query<'a>, ) -> Result { - let vec_row_company = self.get_vec_row_company().await?; - vec_row_company + self.items()? .par_iter() .find_first(|company| company.code.iter().any(|code| code.eq(query.code))) .cloned() @@ -35,8 +34,7 @@ where } async fn get_vec_row_company(&self) -> Result> { - let processed = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?; - Ok(processed) + Ok(self.items()?.to_vec()) } } @@ -48,6 +46,7 @@ mod tests { use crate::domain::repositories::company::queries; use crate::domain::repositories::company::repository::Company; use crate::infrastructure::dsv::Dsv; + use crate::infrastructure::from_slice::FromSlice; use crate::infrastructure::repositories::company::structures::internal::tsv; const TSV: &[u8] = include_bytes!("../../../../assets/companies.tsv"); @@ -68,6 +67,10 @@ mod tests { symbol: Some("13080.0109".to_string()), }; assert_eq!(actual, expected); + let all = tsv::Structure::process_tabular_data(TSV, false)?; + assert_eq!(repository.get_vec_row_company().await?, all); + assert_eq!(repository.get_row_company(&query).await?, expected); + assert_eq!(repository.get_vec_row_company().await?, all); Ok(()) } } diff --git a/src/infrastructure/repositories/statement/dsv.rs b/src/infrastructure/repositories/statement/dsv.rs index 5413dce..2056f5b 100644 --- a/src/infrastructure/repositories/statement/dsv.rs +++ b/src/infrastructure/repositories/statement/dsv.rs @@ -18,15 +18,16 @@ where &self, query: &queries::get_statements::Query<'a>, ) -> Result> { - let processed = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?; + let processed = self.items()?; let vec_row_statement: Vec = match query.code { Some(code) => processed - .into_iter() + .iter() .filter(|statement| statement.code.eq(code)) + .cloned() .collect(), // Noneの場合は、全ての行を返す - None => processed, + None => processed.to_vec(), }; Ok(vec_row_statement) @@ -39,12 +40,29 @@ mod tests { use crate::domain::repositories::statement::queries; use crate::domain::repositories::statement::repository::Statement; use crate::infrastructure::dsv::Dsv; + use crate::infrastructure::from_slice::{DataFormat, FromSlice}; use crate::infrastructure::repositories::statement::structures::internal::tsv; use bytes::Bytes; use chrono::NaiveDate; use serde_json::json; + use std::sync::Barrier; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Instant; const SAMPLE_CODE: &str = "72030"; + static PARSES: [AtomicUsize; 3] = [const { AtomicUsize::new(0) }; 3]; + + struct CountingTsv; + + impl FromSlice for CountingTsv { + type Deserialize = tsv::Structure; + type Item = crate::domain::models::statement::model::RowStatement; + + fn data_format() -> DataFormat { + PARSES[CASE].fetch_add(1, Ordering::SeqCst); + tsv::Structure::data_format() + } + } #[tokio::test] async fn get_row_statements_filters_code_and_preserves_disclosures() -> anyhow::Result<()> { @@ -107,7 +125,10 @@ mod tests { ..Default::default() })?; } - let repository = Dsv::::new(true, Bytes::from(writer.into_inner()?)); + let buffer = Bytes::from(writer.into_inner()?); + let expected = tsv::Structure::process_tabular_data(&buffer, true)?; + let repository = Dsv::>::new(true, buffer.clone()); + assert_eq!(PARSES[0].load(Ordering::SeqCst), 0); let query = queries::get_statements::Query { code: Some(SAMPLE_CODE), }; @@ -152,6 +173,149 @@ mod tests { .await? .is_empty() ); + for code in [Some("13010"), Some(SAMPLE_CODE), None] { + let actual = repository + .get_row_statements(&queries::get_statements::Query { code }) + .await?; + let expected: Vec<_> = expected + .iter() + .filter(|row| code.is_none_or(|code| row.code == code)) + .cloned() + .collect(); + assert_eq!(actual, expected); + } + assert_eq!(PARSES[0].load(Ordering::SeqCst), 1); + let mut returned = repository + .get_row_statements(&queries::get_statements::Query { code: None }) + .await?; + returned[0].code.clear(); + assert_eq!( + repository + .get_row_statements(&queries::get_statements::Query { code: None }) + .await?, + expected + ); + let another = Dsv::>::new(true, buffer); + assert_eq!( + another + .get_row_statements(&queries::get_statements::Query { code: None }) + .await?, + expected + ); + assert_eq!(PARSES[0].load(Ordering::SeqCst), 2); + Ok(()) + } + + #[tokio::test] + async fn invalid_tsv_is_retried_without_caching_errors() -> anyhow::Result<()> { + let mut repository = Dsv::>::new( + true, + Bytes::from_static(b"code\tdisclosed_date\n72030\tinvalid\n"), + ); + let query = queries::get_statements::Query { code: None }; + for expected_calls in 1..=2 { + assert!(repository.get_row_statements(&query).await.is_err()); + assert_eq!(PARSES[1].load(Ordering::SeqCst), expected_calls); + } + repository.buffer = Bytes::from_static(b"code\tdisclosed_date\n72030\t2026-05-08\n"); + assert_eq!(repository.get_row_statements(&query).await?.len(), 1); + assert_eq!(PARSES[1].load(Ordering::SeqCst), 3); + Ok(()) + } + + #[test] + fn concurrent_statement_queries_parse_once() { + let repository = Dsv::>::new( + true, + Bytes::from_static(b"code\tdisclosed_date\n72030\t2026-05-08\n"), + ); + let barrier = Barrier::new(8); + std::thread::scope(|scope| { + for _ in 0..8 { + let repository = &repository; + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + let rows = tokio_test::block_on(repository.get_row_statements( + &queries::get_statements::Query { + code: Some(SAMPLE_CODE), + }, + )) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].code, SAMPLE_CODE); + }); + } + }); + assert_eq!(PARSES[2].load(Ordering::SeqCst), 1); + } + + #[tokio::test] + #[ignore = "manual performance comparison: 10,000 rows and 500 queries"] + async fn benchmark_statement_cache() -> anyhow::Result<()> { + let mut writer = csv::WriterBuilder::new() + .delimiter(b'\t') + .from_writer(Vec::new()); + let codes: Vec<_> = (10000..10500).map(|code| code.to_string()).collect(); + for index in 0..10_000 { + let code = &codes[index % codes.len()]; + writer.serialize(tsv::Structure { + code: code.clone(), + disclosed_date: NaiveDate::from_ymd_opt(2026, 5, 8).unwrap(), + opp: Some(index as f64), + disclosure_json: Some( + json!({ + "code": code, + "disclosed_date": "2026-05-08", + "disclosed_time": "15:00:00", + "disclosure_number": format!("{index:016}"), + "document_type": "FYFinancialStatements_Consolidated_JP", + "current_period_type": "FY", + "current_period_start": "2025-04-01", + "current_period_end": "2026-03-31", + "current_fiscal_year_start": "2025-04-01", + "operating_profit": index as f64, + "operating_profit_period": "unspecified", + "source_note": "x".repeat(512) + }) + .to_string(), + ), + ..Default::default() + })?; + } + let repository = Dsv::::new(true, Bytes::from(writer.into_inner()?)); + let started = Instant::now(); + let mut expected = Vec::new(); + for code in &codes { + // The previous implementation reparsed and then filtered owned rows. + let rows = tsv::Structure::process_tabular_data( + repository.buffer.as_ref(), + repository.has_headers, + )?; + expected.push( + rows.into_iter() + .filter(|row| row.code.eq(code)) + .collect::>(), + ); + } + let uncached = started.elapsed(); + let started = Instant::now(); + let mut actual = Vec::new(); + for code in &codes { + actual.push( + repository + .get_row_statements(&queries::get_statements::Query { code: Some(code) }) + .await?, + ); + } + let cached = started.elapsed(); + assert_eq!(actual, expected); + assert_eq!(actual.iter().map(Vec::len).sum::(), 10_000); + println!( + "rows=10000 queries=500 bytes={} uncached={uncached:?} cached={cached:?} speedup={:.2}x", + repository.buffer.len(), + uncached.as_secs_f64() / cached.as_secs_f64() + ); Ok(()) } } diff --git a/src/infrastructure/repositories/stock/dsv.rs b/src/infrastructure/repositories/stock/dsv.rs index e7ebce6..c7fe376 100644 --- a/src/infrastructure/repositories/stock/dsv.rs +++ b/src/infrastructure/repositories/stock/dsv.rs @@ -21,12 +21,7 @@ where &self, query: &queries::get_stock::Query<'a>, ) -> Result { - let vec_row_stock = self - .get_vec_row_stock(&queries::get_stocks::Query { - ..Default::default() - }) - .await?; - vec_row_stock + self.items()? .par_iter() .find_first(|row_stock| { row_stock @@ -48,17 +43,15 @@ where &self, _: &queries::get_stocks::Query<'a>, ) -> Result> { - let processed = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?; - Ok(processed) + Ok(self.items()?.to_vec()) } async fn get_vec_row_stock_by_date(&self, query: &Query) -> Result> { - let vec_row_stock = self - .get_vec_row_stock(&queries::get_stocks::Query::default()) - .await?; - Ok(vec_row_stock - .into_iter() + Ok(self + .items()? + .iter() .filter(|row_stock| row_stock.date.is_some_and(|date| date.eq(&query.date))) + .cloned() .collect()) } } @@ -72,6 +65,7 @@ mod tests { use crate::domain::repositories::stock::queries; use crate::domain::repositories::stock::repository::Stock; use crate::infrastructure::dsv::Dsv; + use crate::infrastructure::from_slice::FromSlice; use crate::infrastructure::repositories::stock::structures::internal::csv; const CSV: &[u8] = include_bytes!("../../../../assets/8473.T.csv"); @@ -98,6 +92,26 @@ mod tests { adj_volume: Some(3142600.0), }; assert_eq!(actual, expected,); + let all = csv::Structure::process_tabular_data(CSV, true)?; + assert_eq!( + repository + .get_vec_row_stock(&queries::get_stocks::Query::default()) + .await?, + all + ); + let by_date = repository + .get_vec_row_stock_by_date(&queries::get_stocks_by_date::Query { + date: query.target_date, + }) + .await?; + assert_eq!( + by_date, + all.iter() + .filter(|row| row.date == Some(query.target_date)) + .cloned() + .collect::>() + ); + assert_eq!(repository.get_row_stock(&query).await?, expected); Ok(()) }