diff --git a/Cargo.lock b/Cargo.lock index 551b04c..213c19e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1173,7 +1173,7 @@ dependencies = [ [[package]] name = "quantitative-data-analysis-rs" -version = "0.37.4" +version = "0.38.0" dependencies = [ "anyhow", "async-trait", @@ -1205,9 +1205,9 @@ dependencies = [ [[package]] name = "query-string-builder" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "902dd46fd196c802e5b621cce8f5e4d3ff779424b2e37c07662fe88efcf1c7e3" +checksum = "b7556f13bb8d552ecfb2ec546f4df7a6b3f98e421a49b9aa81ae75adee30163e" dependencies = [ "percent-encoding", ] diff --git a/Cargo.toml b/Cargo.toml index 8a22a38..f91e61d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quantitative-data-analysis-rs" -version = "0.37.4" +version = "0.38.0" edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -24,7 +24,7 @@ tokio = { version = "1.48.0", features = ["full"] } async-trait = "0.1.89" tryhard = "0.5.2" reqwest = { version = "0.13.2", features = ["json"] } -query-string-builder = "0.7.1" +query-string-builder = "0.8.0" rayon = "1.11.0" deref-derive = "0.1.0" dirs = "7.0.0" diff --git a/README.md b/README.md index 231f92b..8e92f7a 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,46 @@ regression safety. ## Configuration +### J-Quants financial disclosures + +The J-Quants statement repository retrieves every `/v2/fins/summary` page and +returns disclosure rows in ascending disclosure date and time (JST). Rows with +the same timestamp retain their source order; disclosure numbers do not establish +their chronological order. A failed page, invalid response, or malformed nonempty +field is an error, not a partial history. Rate limiting and retries remain the +caller's responsibility. + +`FetchScoringData.financial_disclosures` contains the API's FY, quarterly, and +forecast-revision rows. Each entry retains its disclosure identifier and document +type, period and fiscal-year dates, and operating-profit actual and forecast +fields, including the `NC` variants. Amounts are in the units provided by +J-Quants (normally yen). Missing values are `null`; they do not mean unchanged or +withdrawn. `FOP` refers to the current fiscal year, `NxFOP` to the next fiscal +year. Compare both fiscal-year boundaries before associating forecasts. + +The API's `OP` is preserved with its original period dates and +`operating_profit_period: "unspecified"`. Its interpretation as cumulative or +single-quarter is not guaranteed here; no single-quarter +differences are calculated. The `NC`-less fields may represent consolidated or +individual values depending on document type; neither accounting basis nor +consolidation status is inferred when the document does not establish it. +J-Quants does not provide the time at which a disclosure became available through +its API, and provider corrections can overwrite previously supplied values. +These records therefore do not guarantee point-in-time reconstruction. +J-Quants also does not guarantee snapshot consistency if data changes during +pagination. + +`latest_statement` and `full_year_sales` remain FY projections, separate from the +full history. `get_row_statements` returns the full disclosure rows, whereas +`get_statements` remains an FY-only projection. Generated statements TSVs now +include a `disclosure_json` column; +`examples/make_statements.rs` writes every disclosure and stops on fetch failure. +Existing TSV data lacks the history and must be regenerated before using it with +`FetchScoringData`; it is rejected rather than silently producing incomplete +history. The JSON contract adds `financial_disclosures` and old JSON must be +regenerated. The example uses `assets/statements.tsv`, so regeneration and +deployment of that input must be handled separately. + ### Stock price and CSV input contract `Stock` requires `adj_open`, `adj_high`, `adj_low`, `adj_close`, and diff --git a/examples/make_statements.rs b/examples/make_statements.rs index 9250d33..fc3c7c5 100644 --- a/examples/make_statements.rs +++ b/examples/make_statements.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use bytes::Bytes; use csv::WriterBuilder; use quantitative_data_analysis_rs::domain::repositories::company::repository::Company; @@ -38,7 +39,6 @@ async fn main() -> anyhow::Result<()> { let started_at = Instant::now(); let total = companies.len(); let mut success_count = 0usize; - let mut failure_count = 0usize; for (index, company) in companies.0.into_iter().enumerate() { let code = company.code; @@ -46,18 +46,14 @@ async fn main() -> anyhow::Result<()> { code: Some(code.as_str()), }; - match jquants_api.get_statements(&query).await { - Ok(statements) => { - for statement in statements.0 { - writer.serialize(statement_tsv::Structure::from(statement))?; - } - success_count += 1; - } - Err(error) => { - eprintln!("failed to fetch statements for code {}: {}", code, error); - failure_count += 1; - } + let statements = jquants_api + .get_row_statements(&query) + .await + .with_context(|| format!("failed to fetch statements for code {code}"))?; + for statement in statements { + writer.serialize(statement_tsv::Structure::try_from(statement)?)?; } + success_count += 1; if (index + 1) % PROGRESS_STEP == 0 || index + 1 == total { let elapsed = started_at.elapsed(); @@ -65,11 +61,10 @@ async fn main() -> anyhow::Result<()> { let remaining = total.saturating_sub(index + 1) as u32; let eta = average_per_item * remaining; eprintln!( - "progress: {}/{} success={} failed={} elapsed={:?} eta={:?}", + "progress: {}/{} success={} elapsed={:?} eta={:?}", index + 1, total, success_count, - failure_count, elapsed, eta, ); diff --git a/src/domain/models/statement/model.rs b/src/domain/models/statement/model.rs index 58dccb2..81b8764 100644 --- a/src/domain/models/statement/model.rs +++ b/src/domain/models/statement/model.rs @@ -1,6 +1,53 @@ -use chrono::NaiveDate; +use chrono::{NaiveDate, NaiveTime}; use deref_derive::{Deref, DerefMut}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; + +/// Period interpretation of the API's actual operating-profit value. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum OperatingProfitPeriod { + /// The API source does not establish a general cumulative or single-quarter meaning. + Unspecified, +} + +/// An unmodified J-Quants disclosure. Times are Japan Standard Time. +/// +/// The API does not explicitly establish whether every operating profit value +/// is cumulative or for a single quarter. Period dates and source fields are +/// retained without deriving or classifying quarterly amounts. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, PartialOrd)] +pub struct FinancialDisclosure { + pub code: String, + pub disclosed_date: NaiveDate, + pub disclosed_time: NaiveTime, + pub disclosure_number: String, + pub document_type: String, + pub current_period_type: Option, + pub current_period_start: Option, + pub current_period_end: Option, + pub current_fiscal_year_start: Option, + pub current_fiscal_year_end: Option, + pub next_fiscal_year_start: Option, + pub next_fiscal_year_end: Option, + pub operating_profit: Option, + pub operating_profit_period: OperatingProfitPeriod, + pub forecast_operating_profit: Option, + pub next_forecast_operating_profit: Option, + pub forecast_operating_profit_2q: Option, + pub next_forecast_operating_profit_2q: Option, + pub non_consolidated_operating_profit: Option, + pub non_consolidated_forecast_operating_profit: Option, + pub non_consolidated_next_forecast_operating_profit: Option, + pub non_consolidated_forecast_operating_profit_2q: Option, + pub non_consolidated_next_forecast_operating_profit_2q: Option, +} + +impl FinancialDisclosure { + /// J-Quants accepts a four-character code and returns its five-character form. + pub fn matches_code(&self, code: &str) -> bool { + self.code == code || (code.len() == 4 && self.code.strip_suffix('0') == Some(code)) + } +} #[derive(Deserialize, Debug, Clone, PartialEq, PartialOrd, Default)] pub struct RowStatement { @@ -16,6 +63,17 @@ pub struct RowStatement { pub profit: Option, pub equity: Option, pub total_assets: Option, + pub disclosure: Option, +} + +impl RowStatement { + /// Retains the legacy FY projection for statement data without disclosure metadata. + pub fn is_usable_full_year(&self) -> bool { + self.disclosure.as_ref().is_none_or(|disclosure| { + disclosure.current_period_type.as_deref() == Some("FY") + && (self.net_sales.is_some() || self.eps.is_some()) + }) + } } /// Represents a financial statement with various financial metrics. @@ -93,6 +151,7 @@ impl TryFrom for Statement { profit, equity, total_assets, + disclosure: _, } = value; Ok(Self { diff --git a/src/domain/repositories/statement/repository.rs b/src/domain/repositories/statement/repository.rs index becdd92..5d4937f 100644 --- a/src/domain/repositories/statement/repository.rs +++ b/src/domain/repositories/statement/repository.rs @@ -12,8 +12,9 @@ pub trait Statement { let row_statements = self.get_row_statements(query).await?; let vec_statement = row_statements .into_iter() - .filter_map(|row_statement| TryFrom::try_from(row_statement).ok()) - .collect(); + .filter(model::RowStatement::is_usable_full_year) + .map(TryFrom::try_from) + .collect::>>()?; Ok(model::Statements(vec_statement)) } diff --git a/src/infrastructure/repositories/statement/dsv.rs b/src/infrastructure/repositories/statement/dsv.rs index b1797db..5413dce 100644 --- a/src/infrastructure/repositories/statement/dsv.rs +++ b/src/infrastructure/repositories/statement/dsv.rs @@ -35,31 +35,123 @@ where #[cfg(test)] mod tests { + use crate::domain::models::statement::model::OperatingProfitPeriod; use crate::domain::repositories::statement::queries; use crate::domain::repositories::statement::repository::Statement; use crate::infrastructure::dsv::Dsv; use crate::infrastructure::repositories::statement::structures::internal::tsv; use bytes::Bytes; use chrono::NaiveDate; + use serde_json::json; const SAMPLE_CODE: &str = "72030"; - const TSV: &[u8] = include_bytes!("../../../../assets/statements.tsv"); #[tokio::test] - async fn get_row_statement_returns_latest() -> anyhow::Result<()> { - let repository = Dsv::::new(true, Bytes::from(TSV)); + async fn get_row_statements_filters_code_and_preserves_disclosures() -> anyhow::Result<()> { + let mut writer = csv::WriterBuilder::new() + .delimiter(b'\t') + .from_writer(Vec::new()); + for (code, date, kind, start, end, fiscal_end, profit) in [ + ( + SAMPLE_CODE, + "2026-05-08", + "FY", + "2025-04-01", + "2026-03-31", + "2026-03-31", + Some(100.0), + ), + ( + "13010", + "2026-05-11", + "FY", + "2025-04-01", + "2026-03-31", + "2026-03-31", + Some(50.0), + ), + ( + SAMPLE_CODE, + "2026-08-07", + "1Q", + "2026-04-01", + "2026-06-30", + "2027-03-31", + None, + ), + ] { + writer.serialize(tsv::Structure { + code: code.into(), + disclosed_date: NaiveDate::parse_from_str(date, "%Y-%m-%d")?, + current_fiscal_year_end_date: Some(NaiveDate::parse_from_str( + fiscal_end, "%Y-%m-%d", + )?), + opp: profit, + disclosure_json: Some( + json!({ + "code": code, + "disclosed_date": date, + "disclosed_time": "15:00:00", + "disclosure_number": format!("{code}-{date}"), + "document_type": format!("{kind}FinancialStatements_Consolidated_JP"), + "current_period_type": kind, + "current_period_start": start, + "current_period_end": end, + "current_fiscal_year_start": start, + "current_fiscal_year_end": fiscal_end, + "operating_profit": profit, + "operating_profit_period": "unspecified" + }) + .to_string(), + ), + ..Default::default() + })?; + } + let repository = Dsv::::new(true, Bytes::from(writer.into_inner()?)); let query = queries::get_statements::Query { code: Some(SAMPLE_CODE), }; let actual = repository.get_row_statements(&query).await?; - assert_eq!(actual[0].code, SAMPLE_CODE); + assert_eq!(actual.len(), 2); + assert!(actual.iter().all(|row| row.code == SAMPLE_CODE)); assert_eq!( actual[0].disclosed_date, - NaiveDate::from_ymd_opt(2022, 5, 11) + NaiveDate::from_ymd_opt(2026, 5, 8) + ); + assert_eq!(actual[0].opp, Some(100.0)); + let annual = actual[0].disclosure.as_ref().unwrap(); + assert_eq!(annual.current_period_type.as_deref(), Some("FY")); + assert_eq!(annual.operating_profit, Some(100.0)); + assert_eq!( + actual[1].disclosed_date, + NaiveDate::from_ymd_opt(2026, 8, 7) + ); + let quarter = actual[1].disclosure.as_ref().unwrap(); + assert_eq!(quarter.current_period_type.as_deref(), Some("1Q")); + assert_eq!( + quarter.current_period_end, + NaiveDate::from_ymd_opt(2026, 6, 30) + ); + assert_eq!(quarter.operating_profit, None); + assert_eq!( + quarter.operating_profit_period, + OperatingProfitPeriod::Unspecified + ); + assert_eq!( + repository + .get_row_statements(&queries::get_statements::Query { code: None }) + .await? + .len(), + 3 + ); + assert!( + repository + .get_row_statements(&queries::get_statements::Query { + code: Some("99990"), + }) + .await? + .is_empty() ); - let actual = actual.last().unwrap(); - assert_eq!(actual.code, SAMPLE_CODE); - assert_eq!(actual.disclosed_date, NaiveDate::from_ymd_opt(2026, 5, 8)); Ok(()) } } diff --git a/src/infrastructure/repositories/statement/jquants_api.rs b/src/infrastructure/repositories/statement/jquants_api.rs index 056e5e6..b5020c4 100644 --- a/src/infrastructure/repositories/statement/jquants_api.rs +++ b/src/infrastructure/repositories/statement/jquants_api.rs @@ -2,55 +2,61 @@ use crate::domain::models::statement::model; use crate::domain::repositories::statement::{queries, repository}; use crate::infrastructure::jquants_api::JQuantsAPI; use crate::infrastructure::repositories::statement::structures::jquants_api::Response; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use async_trait::async_trait; -use query_string_builder::QueryStringOwned; -use rayon::prelude::*; +use query_string_builder::QueryString; use reqwest::Client; use serde_json::Value; +use std::collections::HashSet; +use std::future::Future; const FINS_SUMMARY_URL: &str = "https://api.jquants.com/v2/fins/summary"; -fn has_usable_financial_values(row: &Value) -> bool { - ["Sales", "EPS"].iter().any(|key| { - row[*key] - .as_str() - .is_some_and(|value| !value.trim().is_empty()) - }) +async fn collect_pages(mut fetch_page: F) -> Result> +where + F: FnMut(Option) -> Fut, + Fut: Future>, +{ + let mut rows = Vec::new(); + let mut next = None; + let mut seen = HashSet::new(); + loop { + let response = fetch_page(next).await?; + let page = response + .get("data") + .and_then(Value::as_array) + .context("response data must be an array")?; + rows.extend(page.iter().cloned()); + next = match response.get("pagination_key") { + None => break, + Some(Value::String(key)) if !key.is_empty() => { + if !seen.insert(key.clone()) { + bail!("repeated pagination key"); + } + Some(key.clone()) + } + Some(_) => bail!("invalid pagination key"), + }; + } + Ok(rows) } -fn select_full_year_statements(rows: &[Value]) -> Vec<&Value> { - rows.par_iter() - .filter(|value| { - value["CurPerType"].as_str().is_some_and(|s| s == "FY") - && has_usable_financial_values(value) +fn convert_rows(code: &str, rows: &[Value]) -> Result> { + let mut statements = rows + .iter() + .map(|value| { + model::RowStatement::try_from(Response { + code: code.to_owned(), + value, + }) }) - .collect() -} - -async fn fetch_statement_rows(api_key: &str, code: &str) -> anyhow::Result> { - let qs = QueryStringOwned::new().with("code", code); - let url = format!("{FINS_SUMMARY_URL}{qs}"); - let response = Client::new() - .get(url) - .header("x-api-key", api_key.to_string()) - .send() - .await?; - let response = response.json::().await?; - let Some(rows) = response["data"].as_array() else { - let serialized = serde_json::to_string(&response) - .unwrap_or_else(|_| "".to_string()); - bail!( - "response[data] in response not found. code: {} response: {}", - code, - serialized - ); - }; - if rows.is_empty() { - bail!("response[data] in response is empty array. code: {}", code); - } - - Ok(rows.to_vec()) + .collect::>>()?; + statements.sort_by_key(|row| { + row.disclosure + .as_ref() + .map(|disclosure| (disclosure.disclosed_date, disclosure.disclosed_time)) + }); + Ok(statements) } #[async_trait] @@ -63,96 +69,228 @@ impl repository::Statement for JQuantsAPI { let Some(code) = code else { bail!("code is required for get_row_statements for JQuantsAPI"); }; - let rows = fetch_statement_rows(&self.api_key, code).await?; - let selected = select_full_year_statements(&rows); - if selected.is_empty() { - let serialized = serde_json::to_string_pretty(&rows) - .unwrap_or_else(|_| "".to_string()); - bail!( - "FY statement not found in response[data]. code: {}\n{}", - code, - serialized - ); - } + let client = Client::new(); + let rows = collect_pages(|pagination_key| { + let client = &client; + async move { + let url = { + let mut query = QueryString::new().with("code", code); + if let Some(key) = pagination_key.as_deref() { + query = query.with("pagination_key", key); + } + format!("{FINS_SUMMARY_URL}{query}") + }; + Ok(client + .get(url) + .header("x-api-key", self.api_key.to_string()) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + }) + .await + .with_context(|| format!("failed to fetch disclosures for code {code}"))?; - Ok(selected - .into_iter() - .map(|value| { - model::RowStatement::from(Response { - code: code.to_string(), - value, - }) - }) - .collect()) + convert_rows(code, &rows) } } #[cfg(test)] mod tests { - use crate::domain::models::statement::model; - use crate::domain::repositories::statement::queries; - use crate::domain::repositories::statement::repository::Statement; - use crate::infrastructure::jquants_api::JQuantsAPI; - use crate::infrastructure::repositories::statement::jquants_api::select_full_year_statements; - use crate::shared::jquants_api::setup::Setup; - use anyhow::Result; + use super::{collect_pages, convert_rows}; + use crate::infrastructure::from_slice::FromSlice; + use crate::infrastructure::repositories::statement::structures::internal::tsv::Structure; + use anyhow::anyhow; use chrono::NaiveDate; - use pretty_assertions::assert_eq; - use serde_json::json; + use serde_json::{Value, json}; - #[test] - fn select_full_year_statements_returns_all_fy() { - let rows = vec![ - json!({"CurPerType": "FY", "id": 1, "Sales": "1000", "EPS": "10"}), - json!({"CurPerType": "FY", "id": 2, "Sales": "", "EPS": ""}), - json!({"CurPerType": "FY", "id": 3, "Sales": "1200", "EPS": "12"}), - json!({"CurPerType": "FY", "id": 4, "Sales": "", "EPS": ""}), - ]; - let selected = select_full_year_statements(&rows); - assert_eq!(selected.len(), 2); - assert_eq!(selected[0]["id"], 1); - assert_eq!(selected[1]["id"], 3); + fn disclosure(date: &str, time: &str, number: &str, kind: &str) -> Value { + json!({ + "Code": "12340", "DiscDate": date, "DiscTime": time, "DiscNo": number, + "DocType": "3QFinancialStatements_Consolidated_IFRS", "CurPerType": kind, + "CurPerSt": "2025-04-01", "CurPerEn": "2025-12-31", + "CurFYSt": "2025-04-01", "CurFYEn": "2026-03-31", + "NxtFYSt": "", "NxtFYEn": "", "OP": "90", "FOP": "120", + "NxFOP": "", "NCOP": "70" + }) } #[tokio::test] - async fn get_row_statements_test() -> Result<()> { - let token = Setup::run()?; - let repository = JQuantsAPI::new(token)?; - let query = queries::get_statements::Query { code: Some("8473") }; - let actual = repository.get_row_statements(&query).await?; - let expected = model::RowStatement { - code: "8473".to_string(), - disclosed_date: NaiveDate::from_ymd_opt(2026, 7, 2), - current_fiscal_year_end_date: NaiveDate::from_ymd_opt(2026, 3, 31), - eps: Some(666.82), - bps: Some(2776.99), - annual_dividend_forecast: None, - net_sales: Some(1896607000000.0), - opp: None, - orp: None, - profit: Some(427577000000.0), - equity: Some(2413363000000.0), - total_assets: Some(38290797000000.0), - }; - assert_eq!(actual.last().unwrap().clone(), expected); + async fn collects_every_page_in_order() { + let rows = collect_pages(|key| async move { + match key.as_deref() { + None => Ok(json!({"data": [{"id": 1}], "pagination_key": "next"})), + Some("next") => Ok(json!({"data": [{"id": 2}]})), + _ => Err(anyhow!("unexpected key")), + } + }) + .await + .unwrap(); + assert_eq!(rows, vec![json!({"id": 1}), json!({"id": 2})]); + } - let query = queries::get_statements::Query { code: Some("????") }; - let actual = repository - .get_row_statements(&query) + #[tokio::test] + async fn rejects_failed_or_invalid_later_pages() { + for invalid in [ + json!({"data": null}), + json!({"data": [], "pagination_key": 42}), + json!({"data": [], "pagination_key": null}), + json!({"data": [], "pagination_key": ""}), + ] { + assert!( + collect_pages(|key| { + let invalid = invalid.clone(); + async move { + if key.is_none() { + Ok(json!({"data": [{"id": 1}], "pagination_key": "next"})) + } else { + Ok(invalid) + } + } + }) + .await + .is_err() + ); + } + assert!( + collect_pages(|key| async move { + if key.is_none() { + Ok(json!({"data": [{"id": 1}], "pagination_key": "next"})) + } else { + Err(anyhow!("HTTP 429")) + } + }) .await - .unwrap_err() - .to_string(); - assert!(actual.contains("response[data] in response not found. code: ????")); + .is_err() + ); + assert!( + collect_pages(|_| async { Ok(json!({"data": [{"id": 1}], "pagination_key": "same"})) }) + .await + .is_err() + ); + } - let query = queries::get_statements::Query { code: Some("2995") }; - let actual = repository - .get_row_statements(&query) - .await - .unwrap_err() - .to_string(); - let expected = "response[data] in response is empty array. code: 2995"; - assert_eq!(actual, expected); + #[tokio::test] + async fn preserves_disclosures_across_pages_and_tsv() { + let quarter = disclosure("2026-02-01", "15:00:00", "1", "3Q"); + let earlier = disclosure("2026-02-01", "09:00:00", "9", "2Q"); + let mut annual = disclosure("2026-05-01", "15:00:00", "2", "FY"); + annual["DocType"] = json!("FYFinancialStatements_Consolidated_JP"); + annual["CurPerEn"] = json!("2026-03-31"); + annual["NxtFYSt"] = json!("2026-04-01"); + annual["NxtFYEn"] = json!("2027-03-31"); + annual["FOP"] = json!(""); + annual["NxFOP"] = json!("200"); + let mut revision = disclosure("2026-06-01", "15:00:00", "3", "FY"); + revision["DocType"] = json!("EarnForecastRevision"); + revision["CurFYSt"] = json!("2026-04-01"); + revision["CurFYEn"] = json!("2027-03-31"); + revision["OP"] = json!(""); + revision["FOP"] = json!("210"); + let pages = [ + json!({"data": [quarter, annual], "pagination_key": "next"}), + json!({"data": [earlier, revision]}), + ]; + let rows = collect_pages(|key| { + let page = pages[usize::from(key.is_some())].clone(); + async move { Ok(page) } + }) + .await + .unwrap(); + let statements = convert_rows("1234", &rows).unwrap(); + let disclosures: Vec<_> = statements + .iter() + .map(|row| row.disclosure.as_ref().unwrap()) + .collect(); + assert_eq!( + disclosures + .iter() + .map(|d| d.disclosure_number.as_str()) + .collect::>(), + ["9", "1", "2", "3"] + ); + assert_eq!(disclosures[1].current_period_type.as_deref(), Some("3Q")); + assert_eq!( + disclosures[1].current_period_start, + NaiveDate::from_ymd_opt(2025, 4, 1) + ); + assert_eq!( + disclosures[1].current_period_end, + NaiveDate::from_ymd_opt(2025, 12, 31) + ); + assert_eq!(disclosures[1].operating_profit, Some(90.0)); + assert_eq!( + disclosures[1].operating_profit_period, + crate::domain::models::statement::model::OperatingProfitPeriod::Unspecified + ); + assert_eq!(disclosures[1].non_consolidated_operating_profit, Some(70.0)); + assert_eq!( + disclosures[2].next_fiscal_year_start, + disclosures[3].current_fiscal_year_start + ); + assert_eq!( + disclosures[2].next_fiscal_year_end, + disclosures[3].current_fiscal_year_end + ); + assert_eq!(disclosures[2].next_forecast_operating_profit, Some(200.0)); + assert_eq!(disclosures[3].forecast_operating_profit, Some(210.0)); + assert_eq!(disclosures[3].operating_profit, None); + + let mut writer = csv::WriterBuilder::new() + .delimiter(b'\t') + .from_writer(Vec::new()); + for row in statements.clone() { + writer.serialize(Structure::try_from(row).unwrap()).unwrap(); + } + let bytes = writer.into_inner().unwrap(); + let restored = Structure::process_tabular_data(&bytes, true).unwrap(); + assert_eq!(restored, statements); + let mut mismatched = Structure::try_from(statements[1].clone()).unwrap(); + mismatched.opp = Some(999.0); + assert!( + crate::domain::models::statement::model::RowStatement::try_from(mismatched).is_err() + ); + let mut mismatched = Structure::try_from(statements[1].clone()).unwrap(); + mismatched.code = "9999".into(); + assert!( + crate::domain::models::statement::model::RowStatement::try_from(mismatched).is_err() + ); + let serialized = serde_json::to_value(disclosures[1]).unwrap(); + assert_eq!(serialized["operating_profit_period"], "unspecified"); + assert!( + serialized + .get("derived_single_quarter_operating_profit") + .is_none() + ); + } - Ok(()) + #[test] + fn rejects_invalid_disclosure_fields() { + let valid = disclosure("2026-02-01", "15:00:00", "1", "3Q"); + for (field, invalid) in [ + ("DiscTime", json!("bad")), + ("DiscDate", json!("bad")), + ("DiscNo", json!("")), + ("Code", json!("99990")), + ("OP", json!("NaN")), + ("FOP", json!("bad")), + ("CurPerEn", json!("bad")), + ] { + let mut row = valid.clone(); + row[field] = invalid; + assert!(convert_rows("1234", &[row]).is_err(), "{field}"); + } + let mut missing = valid; + missing["FOP"] = Value::Null; + assert_eq!( + convert_rows("1234", &[missing]).unwrap()[0] + .disclosure + .as_ref() + .unwrap() + .forecast_operating_profit, + None + ); } } diff --git a/src/infrastructure/repositories/statement/structures/internal/tsv.rs b/src/infrastructure/repositories/statement/structures/internal/tsv.rs index 805c811..159109f 100644 --- a/src/infrastructure/repositories/statement/structures/internal/tsv.rs +++ b/src/infrastructure/repositories/statement/structures/internal/tsv.rs @@ -1,3 +1,4 @@ +use anyhow::{Context, Result, ensure}; use chrono::NaiveDate; use serde::{Deserialize, Serialize}; @@ -20,6 +21,8 @@ pub struct Structure { pub profit: Option, pub equity: Option, pub total_assets: Option, + #[serde(default)] + pub disclosure_json: Option, } impl From for Structure { @@ -52,12 +55,41 @@ impl From for Structure { profit: validate_value(profit), equity: validate_value(equity), total_assets: validate_value(total_assets), + disclosure_json: None, } } } -impl From for model::RowStatement { - fn from(value: Structure) -> Self { +impl TryFrom for Structure { + type Error = anyhow::Error; + + fn try_from(value: model::RowStatement) -> Result { + Ok(Self { + code: value.code, + disclosed_date: value.disclosed_date.context("missing disclosed date")?, + current_fiscal_year_end_date: value.current_fiscal_year_end_date, + bps: value.bps, + eps: value.eps, + annual_dividend_forecast: value.annual_dividend_forecast, + net_sales: value.net_sales, + opp: value.opp, + orp: value.orp, + profit: value.profit, + equity: value.equity, + total_assets: value.total_assets, + disclosure_json: value + .disclosure + .as_ref() + .map(serde_json::to_string) + .transpose()?, + }) + } +} + +impl TryFrom for model::RowStatement { + type Error = anyhow::Error; + + fn try_from(value: Structure) -> Result { let Structure { code, disclosed_date, @@ -71,9 +103,31 @@ impl From for model::RowStatement { profit, equity, total_assets, + disclosure_json, } = value; - Self { + let disclosure: Option = disclosure_json + .as_deref() + .filter(|value| !value.is_empty()) + .map(serde_json::from_str) + .transpose() + .context("invalid disclosure_json")?; + if let Some(disclosure) = &disclosure { + ensure!(disclosure.matches_code(&code), "disclosure code mismatch"); + ensure!( + disclosure.disclosed_date == disclosed_date, + "disclosure date mismatch" + ); + ensure!( + disclosure.current_fiscal_year_end == current_fiscal_year_end_date, + "disclosure fiscal year end mismatch" + ); + ensure!( + disclosure.operating_profit == opp, + "disclosure operating profit mismatch" + ); + } + Ok(Self { code, disclosed_date: Some(disclosed_date), current_fiscal_year_end_date, @@ -86,7 +140,8 @@ impl From for model::RowStatement { profit, equity, total_assets, - } + disclosure, + }) } } diff --git a/src/infrastructure/repositories/statement/structures/jquants_api.rs b/src/infrastructure/repositories/statement/structures/jquants_api.rs index 1e16331..514685d 100644 --- a/src/infrastructure/repositories/statement/structures/jquants_api.rs +++ b/src/infrastructure/repositories/statement/structures/jquants_api.rs @@ -1,40 +1,101 @@ -use chrono::NaiveDate; +use anyhow::{Context, Result, bail, ensure}; +use chrono::{NaiveDate, NaiveTime}; use serde_json::Value; +use std::str::FromStr; use crate::domain::models::statement::model; use crate::shared::float::validate_value; -use crate::shared::from_json_string_value::FromJsonStringValue; pub struct Response<'a> { pub code: String, pub value: &'a Value, } -impl From> for model::RowStatement { - fn from(value: Response<'_>) -> Self { - let Response { code, value } = value; - let parse_f64 = |key| { - f64::from_json_string_value(key, value) - .ok() - .and_then(validate_value) - }; +fn optional_string<'a>(value: &'a Value, key: &str) -> Result> { + match value.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) if text.is_empty() => Ok(None), + Some(Value::String(text)) => Ok(Some(text)), + Some(_) => bail!("invalid string field {key}"), + } +} - let disclosed_date = NaiveDate::from_json_string_value("DiscDate", value).ok(); - let current_fiscal_year_end_date = NaiveDate::from_json_string_value("CurFYEn", value).ok(); - let eps = parse_f64("EPS"); - let bps = parse_f64("BPS"); +fn required_string<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + optional_string(value, key)?.with_context(|| format!("missing field {key}")) +} + +fn optional_parsed(value: &Value, key: &str) -> Result> +where + T::Err: std::error::Error + Send + Sync + 'static, +{ + optional_string(value, key)? + .map(|text| text.parse().with_context(|| format!("invalid field {key}"))) + .transpose() +} + +fn optional_number(value: &Value, key: &str) -> Result> { + optional_parsed::(value, key)? + .map(|number| validate_value(number).with_context(|| format!("non-finite field {key}"))) + .transpose() +} + +impl TryFrom> for model::RowStatement { + type Error = anyhow::Error; + + fn try_from(value: Response<'_>) -> Result { + let Response { code, value } = value; + let disclosed_date = required_string(value, "DiscDate")? + .parse::() + .context("invalid field DiscDate")?; + let disclosed_time = required_string(value, "DiscTime")? + .parse::() + .context("invalid field DiscTime")?; + let current_fiscal_year_end_date = optional_parsed(value, "CurFYEn")?; + let eps = optional_number(value, "EPS")?; + let bps = optional_number(value, "BPS")?; // FY は DivAnn(実績配当)に入ることが多く、FDivAnn(予想配当)は空の場合がある。 - let annual_dividend_forecast = parse_f64("DivAnn").or_else(|| parse_f64("FDivAnn")); - let net_sales = parse_f64("Sales"); - let opp = parse_f64("OP"); - let orp = parse_f64("OdP"); - let profit = parse_f64("NP"); - let equity = parse_f64("Eq"); - let total_assets = parse_f64("TA"); + let dividend = optional_number(value, "DivAnn")?; + let forecast_dividend = optional_number(value, "FDivAnn")?; + let annual_dividend_forecast = dividend.or(forecast_dividend); + let net_sales = optional_number(value, "Sales")?; + let opp = optional_number(value, "OP")?; + let orp = optional_number(value, "OdP")?; + let profit = optional_number(value, "NP")?; + let equity = optional_number(value, "Eq")?; + let total_assets = optional_number(value, "TA")?; + let disclosure = model::FinancialDisclosure { + code: required_string(value, "Code")?.to_owned(), + disclosed_date, + disclosed_time, + disclosure_number: required_string(value, "DiscNo")?.to_owned(), + document_type: required_string(value, "DocType")?.to_owned(), + current_period_type: optional_string(value, "CurPerType")?.map(str::to_owned), + current_period_start: optional_parsed(value, "CurPerSt")?, + current_period_end: optional_parsed(value, "CurPerEn")?, + current_fiscal_year_start: optional_parsed(value, "CurFYSt")?, + current_fiscal_year_end: current_fiscal_year_end_date, + next_fiscal_year_start: optional_parsed(value, "NxtFYSt")?, + next_fiscal_year_end: optional_parsed(value, "NxtFYEn")?, + operating_profit: opp, + operating_profit_period: model::OperatingProfitPeriod::Unspecified, + forecast_operating_profit: optional_number(value, "FOP")?, + next_forecast_operating_profit: optional_number(value, "NxFOP")?, + forecast_operating_profit_2q: optional_number(value, "FOP2Q")?, + next_forecast_operating_profit_2q: optional_number(value, "NxFOP2Q")?, + non_consolidated_operating_profit: optional_number(value, "NCOP")?, + non_consolidated_forecast_operating_profit: optional_number(value, "FNCOP")?, + non_consolidated_next_forecast_operating_profit: optional_number(value, "NxFNCOP")?, + non_consolidated_forecast_operating_profit_2q: optional_number(value, "FNCOP2Q")?, + non_consolidated_next_forecast_operating_profit_2q: optional_number( + value, + "NxFNCOP2Q", + )?, + }; - Self { + ensure!(disclosure.matches_code(&code), "disclosure code mismatch"); + Ok(Self { code, - disclosed_date, + disclosed_date: Some(disclosed_date), current_fiscal_year_end_date, eps, bps, @@ -45,7 +106,8 @@ impl From> for model::RowStatement { profit, equity, total_assets, - } + disclosure: Some(disclosure), + }) } } @@ -60,15 +122,17 @@ mod tests { fn annual_dividend_forecast_uses_divann_first() { let value = json!({ "DiscDate": "2026-05-09", + "DiscTime": "15:00:00", "Code": "84730", "DiscNo": "1", "DocType": "FYFinancialStatements_Consolidated_JP", "CurFYEn": "2026-03-31", "DivAnn": "170.0", "FDivAnn": "" }); - let actual = model::RowStatement::from(Response { + let actual = model::RowStatement::try_from(Response { code: "8473".to_string(), value: &value, - }); + }) + .unwrap(); assert_eq!( actual.current_fiscal_year_end_date, NaiveDate::from_ymd_opt(2026, 3, 31) @@ -80,15 +144,17 @@ mod tests { fn annual_dividend_forecast_falls_back_to_fdivann() { let value = json!({ "DiscDate": "2024-02-07", + "DiscTime": "15:00:00", "Code": "84730", "DiscNo": "1", "DocType": "3QFinancialStatements_Consolidated_JP", "CurFYEn": "2024-03-31", "DivAnn": "", "FDivAnn": "160.0" }); - let actual = model::RowStatement::from(Response { + let actual = model::RowStatement::try_from(Response { code: "8473".to_string(), value: &value, - }); + }) + .unwrap(); assert_eq!( actual.current_fiscal_year_end_date, NaiveDate::from_ymd_opt(2024, 3, 31) diff --git a/src/presenter/presenters/fetch_scoring_data/output.rs b/src/presenter/presenters/fetch_scoring_data/output.rs index 139822b..bad8fdb 100644 --- a/src/presenter/presenters/fetch_scoring_data/output.rs +++ b/src/presenter/presenters/fetch_scoring_data/output.rs @@ -1,3 +1,4 @@ +use crate::domain::models::statement::model::FinancialDisclosure; use chrono::{DateTime, NaiveDate, Utc}; use serde::{Deserialize, Serialize}; @@ -11,6 +12,7 @@ pub struct FetchScoringData { pub price: Option, pub latest_statement: Option, pub full_year_sales: Vec, + pub financial_disclosures: Vec, } #[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] diff --git a/src/use_case/interactors/fetch_scoring_data/interactor.rs b/src/use_case/interactors/fetch_scoring_data/interactor.rs index 5f47a43..2442621 100644 --- a/src/use_case/interactors/fetch_scoring_data/interactor.rs +++ b/src/use_case/interactors/fetch_scoring_data/interactor.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Result, ensure}; use async_trait::async_trait; use crate::domain::models::company::model as company_model; @@ -55,6 +55,7 @@ where price: None, latest_statement: None, full_year_sales: vec![], + financial_disclosures: vec![], }); }; @@ -74,17 +75,14 @@ where price: None, latest_statement: None, full_year_sales: vec![], + financial_disclosures: vec![], }); }; let query = get_statements::Query { code: Some(input.code.as_str()), }; - let mut row_statements = self - .statement_repository - .get_row_statements(&query) - .await - .unwrap_or_else(|_| vec![]); + let mut row_statements = self.statement_repository.get_row_statements(&query).await?; if row_statements.is_empty() { return Ok(output::FetchScoringData { @@ -96,17 +94,33 @@ where price: Some(output::FetchPrice::from(row_stock)), latest_statement: None, full_year_sales: vec![], + financial_disclosures: vec![], }); } + ensure!( + row_statements.iter().all(|row| row.disclosure.is_some()), + "financial disclosure history is missing for code {}; regenerate the statement data", + input.code + ); - row_statements.sort_by(|a, b| { - let a_date = a.disclosed_date.unwrap_or_default(); - let b_date = b.disclosed_date.unwrap_or_default(); - a_date.cmp(&b_date) + row_statements.sort_by_key(|row| { + ( + row.disclosed_date, + row.disclosure + .as_ref() + .map(|disclosure| disclosure.disclosed_time), + ) }); - let latest_statement = row_statements.last().unwrap().clone(); - - let full_year_sales = row_statements + let financial_disclosures = row_statements + .iter() + .filter_map(|row| row.disclosure.clone()) + .collect(); + let full_year_statements = row_statements + .into_iter() + .filter(statement_model::RowStatement::is_usable_full_year) + .collect::>(); + let latest_statement = full_year_statements.last().cloned(); + let full_year_sales = full_year_statements .into_iter() .map(output::FetchFullYearSales::from) .collect::>(); @@ -118,8 +132,9 @@ where error_type: None, company: Some(output::FetchCompany::from(row_company)), price: Some(output::FetchPrice::from(row_stock)), - latest_statement: Some(output::FetchLatestStatement::from(latest_statement)), + latest_statement: latest_statement.map(output::FetchLatestStatement::from), full_year_sales, + financial_disclosures, }) } } @@ -162,3 +177,157 @@ impl From for output::FetchFullYearSales { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::infrastructure::repositories::statement::structures::jquants_api::Response; + use chrono::{NaiveDate, Utc}; + use serde_json::json; + + struct Company; + struct Stock; + struct Statements { + rows: Vec, + fail: bool, + } + + #[async_trait] + impl repositories::company::repository::Company for Company { + async fn get_row_company<'a>( + &self, + _: &get_company::Query<'a>, + ) -> Result { + Ok(company_model::RowCompany::default()) + } + + async fn get_vec_row_company(&self) -> Result> { + Ok(vec![]) + } + } + + #[async_trait] + impl repositories::stock::repository::Stock for Stock { + async fn get_row_stock<'a>( + &self, + _: &get_stock::Query<'a>, + ) -> Result { + Ok(stock_model::RowStock::default()) + } + + async fn get_vec_row_stock<'a>( + &self, + _: &repositories::stock::queries::get_stocks::Query<'a>, + ) -> Result> { + Ok(vec![]) + } + + async fn get_vec_row_stock_by_date( + &self, + _: &repositories::stock::queries::get_stocks_by_date::Query, + ) -> Result> { + Ok(vec![]) + } + } + + #[async_trait] + impl repositories::statement::repository::Statement for Statements { + async fn get_row_statements<'a>( + &self, + _: &get_statements::Query<'a>, + ) -> Result> { + if self.fail { + anyhow::bail!("page fetch failed"); + } + Ok(self.rows.clone()) + } + } + + fn row( + date: &str, + time: &str, + number: &str, + period: &str, + sales: &str, + ) -> statement_model::RowStatement { + let value = json!({ + "Code": "12340", "DiscDate": date, "DiscTime": time, "DiscNo": number, + "DocType": "FinancialStatements_Consolidated_JP", "CurPerType": period, + "CurFYEn": "2026-03-31", "Sales": sales, "OP": "90", "FOP": "" + }); + statement_model::RowStatement::try_from(Response { + code: "1234".into(), + value: &value, + }) + .unwrap() + } + + fn input() -> input::FetchScoringData { + input::FetchScoringData::new( + "1234".into(), + NaiveDate::from_ymd_opt(2026, 8, 1).unwrap(), + Utc::now(), + ) + } + + #[tokio::test] + async fn history_is_sorted_but_annual_projections_exclude_quarters() { + use crate::use_case::interfaces::fetch_scoring_data::use_case::FetchScoringData as _; + let statements = Statements { + rows: vec![ + row("2026-08-01", "15:00:00", "1", "1Q", "25"), + row("2026-05-01", "15:00:00", "2", "FY", "100"), + row("2026-08-01", "09:00:00", "9", "1Q", "20"), + ], + fail: false, + }; + let result = FetchScoringData::new(&Company, &Stock, &statements) + .handle(input()) + .await + .unwrap(); + assert_eq!( + result + .financial_disclosures + .iter() + .map(|d| d.disclosure_number.as_str()) + .collect::>(), + ["2", "9", "1"] + ); + assert_eq!(result.full_year_sales.len(), 1); + assert_eq!(result.full_year_sales[0].net_sales, Some(100.0)); + assert_eq!( + result.latest_statement.unwrap().disclosed_date, + NaiveDate::from_ymd_opt(2026, 5, 1) + ); + let json = serde_json::to_value(result.financial_disclosures).unwrap(); + assert_eq!( + json[2]["forecast_operating_profit"], + serde_json::Value::Null + ); + } + + #[tokio::test] + async fn repository_failures_and_missing_history_are_errors() { + use crate::use_case::interfaces::fetch_scoring_data::use_case::FetchScoringData as _; + let failing = Statements { + rows: vec![], + fail: true, + }; + assert!( + FetchScoringData::new(&Company, &Stock, &failing) + .handle(input()) + .await + .is_err() + ); + let missing = Statements { + rows: vec![statement_model::RowStatement::default()], + fail: false, + }; + assert!( + FetchScoringData::new(&Company, &Stock, &missing) + .handle(input()) + .await + .is_err() + ); + } +} diff --git a/src/use_case/interactors/score_stock/interactor.rs b/src/use_case/interactors/score_stock/interactor.rs index 5773e1d..0e0ae73 100644 --- a/src/use_case/interactors/score_stock/interactor.rs +++ b/src/use_case/interactors/score_stock/interactor.rs @@ -181,6 +181,7 @@ mod tests { net_sales: Some(100.0), }, ], + financial_disclosures: vec![], } }