Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "quantitative-data-analysis-rs"
version = "0.38.0"
version = "0.39.0"
edition = "2024"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
Expand Down
28 changes: 28 additions & 0 deletions src/infrastructure/dsv.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
use bytes::Bytes;
use std::fmt::{Debug, Display};
use std::marker::PhantomData;
use std::sync::{Mutex, OnceLock};

use crate::infrastructure::from_slice::FromSlice;

/// Delimited data parsed lazily and cached per instance after a successful parse.
///
/// Do not modify `buffer` or `has_headers` after construction. Changes to these
/// public fields do not invalidate the cached items.
pub struct Dsv<T: FromSlice> {
pub has_headers: bool,
pub buffer: Bytes,
_structure: PhantomData<T>,
parsed_items: OnceLock<Vec<T::Item>>,
initialization: Mutex<()>,
}

impl<T: FromSlice> Dsv<T> {
Expand All @@ -15,6 +23,26 @@ impl<T: FromSlice> Dsv<T> {
has_headers,
buffer,
_structure: PhantomData,
parsed_items: OnceLock::new(),
initialization: Mutex::new(()),
}
}

pub(crate) fn items(&self) -> anyhow::Result<&[T::Item]>
where
<T::Item as TryFrom<T::Deserialize>>::Error: Debug + Display,
{
if let Some(items) = self.parsed_items.get() {
return Ok(items);
}
let _guard = self
.initialization
.lock()
.map_err(|error| anyhow::anyhow!("DSV cache initialization lock poisoned: {error}"))?;
if let Some(items) = self.parsed_items.get() {
return Ok(items);
}
let items = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?;
Ok(self.parsed_items.get_or_init(|| items))
}
}
11 changes: 7 additions & 4 deletions src/infrastructure/repositories/company/dsv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ where
&self,
query: &queries::get_company::Query<'a>,
) -> Result<model::RowCompany> {
let vec_row_company = self.get_vec_row_company().await?;
vec_row_company
self.items()?
.par_iter()
.find_first(|company| company.code.iter().any(|code| code.eq(query.code)))
.cloned()
Expand All @@ -35,8 +34,7 @@ where
}

async fn get_vec_row_company(&self) -> Result<Vec<model::RowCompany>> {
let processed = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?;
Ok(processed)
Ok(self.items()?.to_vec())
}
}

Expand All @@ -48,6 +46,7 @@ mod tests {
use crate::domain::repositories::company::queries;
use crate::domain::repositories::company::repository::Company;
use crate::infrastructure::dsv::Dsv;
use crate::infrastructure::from_slice::FromSlice;
use crate::infrastructure::repositories::company::structures::internal::tsv;

const TSV: &[u8] = include_bytes!("../../../../assets/companies.tsv");
Expand All @@ -68,6 +67,10 @@ mod tests {
symbol: Some("13080.0109".to_string()),
};
assert_eq!(actual, expected);
let all = tsv::Structure::process_tabular_data(TSV, false)?;
assert_eq!(repository.get_vec_row_company().await?, all);
assert_eq!(repository.get_row_company(&query).await?, expected);
assert_eq!(repository.get_vec_row_company().await?, all);
Ok(())
}
}
172 changes: 168 additions & 4 deletions src/infrastructure/repositories/statement/dsv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,16 @@ where
&self,
query: &queries::get_statements::Query<'a>,
) -> Result<Vec<model::RowStatement>> {
let processed = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?;
let processed = self.items()?;

let vec_row_statement: Vec<model::RowStatement> = match query.code {
Some(code) => processed
.into_iter()
.iter()
.filter(|statement| statement.code.eq(code))
.cloned()
.collect(),
// Noneの場合は、全ての行を返す
None => processed,
None => processed.to_vec(),
};

Ok(vec_row_statement)
Expand All @@ -39,12 +40,29 @@ mod tests {
use crate::domain::repositories::statement::queries;
use crate::domain::repositories::statement::repository::Statement;
use crate::infrastructure::dsv::Dsv;
use crate::infrastructure::from_slice::{DataFormat, FromSlice};
use crate::infrastructure::repositories::statement::structures::internal::tsv;
use bytes::Bytes;
use chrono::NaiveDate;
use serde_json::json;
use std::sync::Barrier;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Instant;

const SAMPLE_CODE: &str = "72030";
static PARSES: [AtomicUsize; 3] = [const { AtomicUsize::new(0) }; 3];

struct CountingTsv<const CASE: usize>;

impl<const CASE: usize> FromSlice for CountingTsv<CASE> {
type Deserialize = tsv::Structure;
type Item = crate::domain::models::statement::model::RowStatement;

fn data_format() -> DataFormat {
PARSES[CASE].fetch_add(1, Ordering::SeqCst);
tsv::Structure::data_format()
}
}

#[tokio::test]
async fn get_row_statements_filters_code_and_preserves_disclosures() -> anyhow::Result<()> {
Expand Down Expand Up @@ -107,7 +125,10 @@ mod tests {
..Default::default()
})?;
}
let repository = Dsv::<tsv::Structure>::new(true, Bytes::from(writer.into_inner()?));
let buffer = Bytes::from(writer.into_inner()?);
let expected = tsv::Structure::process_tabular_data(&buffer, true)?;
let repository = Dsv::<CountingTsv<0>>::new(true, buffer.clone());
assert_eq!(PARSES[0].load(Ordering::SeqCst), 0);
let query = queries::get_statements::Query {
code: Some(SAMPLE_CODE),
};
Expand Down Expand Up @@ -152,6 +173,149 @@ mod tests {
.await?
.is_empty()
);
for code in [Some("13010"), Some(SAMPLE_CODE), None] {
let actual = repository
.get_row_statements(&queries::get_statements::Query { code })
.await?;
let expected: Vec<_> = expected
.iter()
.filter(|row| code.is_none_or(|code| row.code == code))
.cloned()
.collect();
assert_eq!(actual, expected);
}
assert_eq!(PARSES[0].load(Ordering::SeqCst), 1);
let mut returned = repository
.get_row_statements(&queries::get_statements::Query { code: None })
.await?;
returned[0].code.clear();
assert_eq!(
repository
.get_row_statements(&queries::get_statements::Query { code: None })
.await?,
expected
);
let another = Dsv::<CountingTsv<0>>::new(true, buffer);
assert_eq!(
another
.get_row_statements(&queries::get_statements::Query { code: None })
.await?,
expected
);
assert_eq!(PARSES[0].load(Ordering::SeqCst), 2);
Ok(())
}

#[tokio::test]
async fn invalid_tsv_is_retried_without_caching_errors() -> anyhow::Result<()> {
let mut repository = Dsv::<CountingTsv<1>>::new(
true,
Bytes::from_static(b"code\tdisclosed_date\n72030\tinvalid\n"),
);
let query = queries::get_statements::Query { code: None };
for expected_calls in 1..=2 {
assert!(repository.get_row_statements(&query).await.is_err());
assert_eq!(PARSES[1].load(Ordering::SeqCst), expected_calls);
}
repository.buffer = Bytes::from_static(b"code\tdisclosed_date\n72030\t2026-05-08\n");
assert_eq!(repository.get_row_statements(&query).await?.len(), 1);
assert_eq!(PARSES[1].load(Ordering::SeqCst), 3);
Ok(())
}

#[test]
fn concurrent_statement_queries_parse_once() {
let repository = Dsv::<CountingTsv<2>>::new(
true,
Bytes::from_static(b"code\tdisclosed_date\n72030\t2026-05-08\n"),
);
let barrier = Barrier::new(8);
std::thread::scope(|scope| {
for _ in 0..8 {
let repository = &repository;
let barrier = &barrier;
scope.spawn(move || {
barrier.wait();
let rows = tokio_test::block_on(repository.get_row_statements(
&queries::get_statements::Query {
code: Some(SAMPLE_CODE),
},
))
.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].code, SAMPLE_CODE);
});
}
});
assert_eq!(PARSES[2].load(Ordering::SeqCst), 1);
}

#[tokio::test]
#[ignore = "manual performance comparison: 10,000 rows and 500 queries"]
async fn benchmark_statement_cache() -> anyhow::Result<()> {
let mut writer = csv::WriterBuilder::new()
.delimiter(b'\t')
.from_writer(Vec::new());
let codes: Vec<_> = (10000..10500).map(|code| code.to_string()).collect();
for index in 0..10_000 {
let code = &codes[index % codes.len()];
writer.serialize(tsv::Structure {
code: code.clone(),
disclosed_date: NaiveDate::from_ymd_opt(2026, 5, 8).unwrap(),
opp: Some(index as f64),
disclosure_json: Some(
json!({
"code": code,
"disclosed_date": "2026-05-08",
"disclosed_time": "15:00:00",
"disclosure_number": format!("{index:016}"),
"document_type": "FYFinancialStatements_Consolidated_JP",
"current_period_type": "FY",
"current_period_start": "2025-04-01",
"current_period_end": "2026-03-31",
"current_fiscal_year_start": "2025-04-01",
"operating_profit": index as f64,
"operating_profit_period": "unspecified",
"source_note": "x".repeat(512)
})
.to_string(),
),
..Default::default()
})?;
}
let repository = Dsv::<tsv::Structure>::new(true, Bytes::from(writer.into_inner()?));
let started = Instant::now();
let mut expected = Vec::new();
for code in &codes {
// The previous implementation reparsed and then filtered owned rows.
let rows = tsv::Structure::process_tabular_data(
repository.buffer.as_ref(),
repository.has_headers,
)?;
expected.push(
rows.into_iter()
.filter(|row| row.code.eq(code))
.collect::<Vec<_>>(),
);
}
let uncached = started.elapsed();
let started = Instant::now();
let mut actual = Vec::new();
for code in &codes {
actual.push(
repository
.get_row_statements(&queries::get_statements::Query { code: Some(code) })
.await?,
);
}
let cached = started.elapsed();
assert_eq!(actual, expected);
assert_eq!(actual.iter().map(Vec::len).sum::<usize>(), 10_000);
println!(
"rows=10000 queries=500 bytes={} uncached={uncached:?} cached={cached:?} speedup={:.2}x",
repository.buffer.len(),
uncached.as_secs_f64() / cached.as_secs_f64()
);
Ok(())
}
}
40 changes: 27 additions & 13 deletions src/infrastructure/repositories/stock/dsv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,7 @@ where
&self,
query: &queries::get_stock::Query<'a>,
) -> Result<model::RowStock> {
let vec_row_stock = self
.get_vec_row_stock(&queries::get_stocks::Query {
..Default::default()
})
.await?;
vec_row_stock
self.items()?
.par_iter()
.find_first(|row_stock| {
row_stock
Expand All @@ -48,17 +43,15 @@ where
&self,
_: &queries::get_stocks::Query<'a>,
) -> Result<Vec<model::RowStock>> {
let processed = T::process_tabular_data(self.buffer.as_ref(), self.has_headers)?;
Ok(processed)
Ok(self.items()?.to_vec())
}

async fn get_vec_row_stock_by_date(&self, query: &Query) -> Result<Vec<model::RowStock>> {
let vec_row_stock = self
.get_vec_row_stock(&queries::get_stocks::Query::default())
.await?;
Ok(vec_row_stock
.into_iter()
Ok(self
.items()?
.iter()
.filter(|row_stock| row_stock.date.is_some_and(|date| date.eq(&query.date)))
.cloned()
.collect())
}
}
Expand All @@ -72,6 +65,7 @@ mod tests {
use crate::domain::repositories::stock::queries;
use crate::domain::repositories::stock::repository::Stock;
use crate::infrastructure::dsv::Dsv;
use crate::infrastructure::from_slice::FromSlice;
use crate::infrastructure::repositories::stock::structures::internal::csv;

const CSV: &[u8] = include_bytes!("../../../../assets/8473.T.csv");
Expand All @@ -98,6 +92,26 @@ mod tests {
adj_volume: Some(3142600.0),
};
assert_eq!(actual, expected,);
let all = csv::Structure::process_tabular_data(CSV, true)?;
assert_eq!(
repository
.get_vec_row_stock(&queries::get_stocks::Query::default())
.await?,
all
);
let by_date = repository
.get_vec_row_stock_by_date(&queries::get_stocks_by_date::Query {
date: query.target_date,
})
.await?;
assert_eq!(
by_date,
all.iter()
.filter(|row| row.date == Some(query.target_date))
.cloned()
.collect::<Vec<_>>()
);
assert_eq!(repository.get_row_stock(&query).await?, expected);
Ok(())
}

Expand Down
Loading